diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index eea3507e5..201cee7e2 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -49,8 +49,14 @@ jobs: - uses: actions/checkout@v3 with: submodules: true + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + java-version: 8 + distribution: 'temurin' + cache: 'maven' - name: Check code style - run: ./mvnw --batch-mode --quiet --no-snapshot-updates clean checkstyle:check + run: ./mvnw --batch-mode --quiet --no-snapshot-updates clean spotless:check dead-link: if: github.repository == 'apache/incubator-seatunnel-web' diff --git a/pom.xml b/pom.xml index c015a38de..ef0fc19a7 100644 --- a/pom.xml +++ b/pom.xml @@ -13,8 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - 4.0.0 @@ -26,55 +25,17 @@ org.apache.seatunnel seatunnel-web - pom ${revision} + pom SeaTunnel - - Production ready big data processing product based on Apache Spark and Apache Flink. - - - https://github.com/apache/incubator-seatunnel - - - - The Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - - - - - scm:git:https://github.com/apache/incubator-seatunnel.git - scm:git:https://github.com/apache/incubator-seatunnel.git - https://github.com/apache/incubator-seatunnel - HEAD - - - - GitHub - https://github.com/apache/incubator-seatunnel/issues - - - - - SeaTunnel Developer List - dev@seatunnel.apache.org - dev-subscribe@seatunnel.apache.org - dev-unsubscribe@seatunnel.apache.org - - - SeaTunnel Commits List - commits@seatunnel.apache.org - commits-subscribe@seatunnel.apache.org - commits-unsubscribe@seatunnel.apache.org - - + Production ready big data processing product based on Apache Spark and Apache Flink. - seatunnel-server - seatunnel-datasource - seatunnel-web-dist + seatunnel-server + seatunnel-datasource + seatunnel-web-dist @@ -143,6 +104,7 @@ 2.3.1 3.1.4 1.11.271 + 2.29.0 @@ -154,24 +116,24 @@ ${seatunnel-framework.version} - jcl-over-slf4j org.slf4j + jcl-over-slf4j - log4j-1.2-api org.apache.logging.log4j + log4j-1.2-api - log4j-api org.apache.logging.log4j + log4j-api - log4j-core org.apache.logging.log4j + log4j-core - log4j-slf4j-impl org.apache.logging.log4j + log4j-slf4j-impl @@ -187,28 +149,28 @@ jackson-dataformat-properties - slf4j-log4j12 org.slf4j + slf4j-log4j12 - log4j-1.2-api org.apache.logging.log4j + log4j-1.2-api - jcl-over-slf4j org.slf4j + jcl-over-slf4j - log4j-api org.apache.logging.log4j + log4j-api - log4j-core org.apache.logging.log4j + log4j-core - log4j-slf4j-impl org.apache.logging.log4j + log4j-slf4j-impl @@ -219,24 +181,24 @@ ${seatunnel-framework.version} - jcl-over-slf4j org.slf4j + jcl-over-slf4j - log4j-1.2-api org.apache.logging.log4j + log4j-1.2-api - log4j-api org.apache.logging.log4j + log4j-api - log4j-core org.apache.logging.log4j + log4j-core - log4j-slf4j-impl org.apache.logging.log4j + log4j-slf4j-impl @@ -260,24 +222,24 @@ ${seatunnel-framework.version} - jcl-over-slf4j org.slf4j + jcl-over-slf4j - log4j-1.2-api org.apache.logging.log4j + log4j-1.2-api - log4j-api org.apache.logging.log4j + log4j-api - log4j-core org.apache.logging.log4j + log4j-core - log4j-slf4j-impl org.apache.logging.log4j + log4j-slf4j-impl @@ -344,8 +306,8 @@ spring-boot-starter-jdbc - spring-boot-autoconfigure org.springframework.boot + spring-boot-autoconfigure @@ -502,8 +464,8 @@ org.apache.seatunnel seatunnel-hadoop3-3.1.4-uber - provided ${hadoop-uber.version} + provided org.apache.avro @@ -535,8 +497,8 @@ ${cron-utils.version} - javassist org.javassist + javassist @@ -593,8 +555,7 @@ ${skipUT} - ${project.build.directory}/jacoco.exec - + ${project.build.directory}/jacoco.exec **/*IT.java @@ -602,7 +563,6 @@ - org.apache.maven.plugins @@ -664,8 +624,7 @@ ${maven-checkstyle-plugin.version} - ${maven.multiModuleProjectDirectory}/tools/checkstyle/checkStyle.xml - + ${maven.multiModuleProjectDirectory}/tools/checkstyle/checkStyle.xml UTF-8 true true @@ -674,28 +633,23 @@ ${project.build.sourceDirectory} ${project.build.testSourceDirectory} - - **/*.properties, + **/*.properties, **/*.sh, **/*.bat, **/*.yml, **/*.yaml, - **/*.xml - - - **/.asf.yaml, - **/.github/** - - - + **/*.xml + **/.asf.yaml, + **/.github/** + validate - process-sources check + process-sources @@ -737,7 +691,6 @@ - org.codehaus.mojo build-helper-maven-plugin @@ -791,6 +744,81 @@ maven-dependency-plugin ${maven-dependency-plugin.version} + + + com.diffplug.spotless + spotless-maven-plugin + ${spotless.version} + + + + 1.7 + + + + + + org.apache.seatunnel.shade,org.apache.seatunnel,org.apache,org,,javax,java,\# + + + Remove wildcard imports + import\s+(static)*\s*[^\*\s]+\*;(\r\n|\r|\n) + $1 + + + Block powermock + import\s+org\.powermock\.[^\*\s]*(|\*);(\r\n|\r|\n) + $1 + + + Block jUnit4 imports + import\s+org\.junit\.[^jupiter][^\*\s]*(|\*);(\r\n|\r|\n) + $1 + + + + + UTF-8 + 4 + true + false + true + true + false + false + custom_1 + false + false + + + Leading blank line + project + project + + + + + docs/**/*.md + + + **/.github/**/*.md + + + + + true + + + + + + check + + compile + + + + @@ -837,7 +865,48 @@ org.codehaus.mojo license-maven-plugin + + + com.diffplug.spotless + spotless-maven-plugin + + https://github.com/apache/incubator-seatunnel + + + + The Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + + + + + SeaTunnel Developer List + dev-subscribe@seatunnel.apache.org + dev-unsubscribe@seatunnel.apache.org + dev@seatunnel.apache.org + + + SeaTunnel Commits List + commits-subscribe@seatunnel.apache.org + commits-unsubscribe@seatunnel.apache.org + commits@seatunnel.apache.org + + + + + scm:git:https://github.com/apache/incubator-seatunnel.git + scm:git:https://github.com/apache/incubator-seatunnel.git + https://github.com/apache/incubator-seatunnel + HEAD + + + + GitHub + https://github.com/apache/incubator-seatunnel/issues + + diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java index 7cc2783ed..b9e3b4f33 100644 --- a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java +++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java @@ -17,8 +17,6 @@ package org.apache.seatunnel.datasource; -import static com.google.common.base.Preconditions.checkNotNull; - import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.exception.DataSourceSDKException; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; @@ -34,6 +32,8 @@ import java.util.ServiceLoader; import java.util.concurrent.atomic.AtomicInteger; +import static com.google.common.base.Preconditions.checkNotNull; + public abstract class AbstractDataSourceClient implements DataSourceService { private Map supportedDataSourceInfo = new HashMap<>(); @@ -47,23 +47,23 @@ public abstract class AbstractDataSourceClient implements DataSourceService { protected AbstractDataSourceClient() { AtomicInteger dataSourceIndex = new AtomicInteger(); ServiceLoader.load(DataSourceFactory.class) - .forEach( - seaTunnelDataSourceFactory -> { - seaTunnelDataSourceFactory - .supportedDataSources() - .forEach( - dataSourceInfo -> { - supportedDataSourceInfo.put( - dataSourceInfo.getName().toUpperCase(), - dataSourceInfo); - supportedDataSourceIndex.put( - dataSourceInfo.getName().toUpperCase(), - dataSourceIndex.get()); - supportedDataSources.add(dataSourceInfo); - }); - dataSourceChannels.add(seaTunnelDataSourceFactory.createChannel()); - dataSourceIndex.getAndIncrement(); - }); + .forEach( + seaTunnelDataSourceFactory -> { + seaTunnelDataSourceFactory + .supportedDataSources() + .forEach( + dataSourceInfo -> { + supportedDataSourceInfo.put( + dataSourceInfo.getName().toUpperCase(), + dataSourceInfo); + supportedDataSourceIndex.put( + dataSourceInfo.getName().toUpperCase(), + dataSourceIndex.get()); + supportedDataSources.add(dataSourceInfo); + }); + dataSourceChannels.add(seaTunnelDataSourceFactory.createChannel()); + dataSourceIndex.getAndIncrement(); + }); if (supportedDataSourceInfo.isEmpty()) { throw new DataSourceSDKException("No supported data source found"); } @@ -71,9 +71,9 @@ protected AbstractDataSourceClient() { @Override public Boolean checkDataSourceConnectivity( - String pluginName, Map dataSourceParams) { + String pluginName, Map dataSourceParams) { return getDataSourceChannel(pluginName) - .checkDataSourceConnectivity(pluginName, dataSourceParams); + .checkDataSourceConnectivity(pluginName, dataSourceParams); } @Override @@ -86,7 +86,7 @@ protected DataSourceChannel getDataSourceChannel(String pluginName) { Integer index = supportedDataSourceIndex.get(pluginName.toUpperCase()); if (index == null) { throw new DataSourceSDKException( - "The %s plugin is not supported or plugin not exist.", pluginName); + "The %s plugin is not supported or plugin not exist.", pluginName); } return dataSourceChannels.get(index); } @@ -99,12 +99,12 @@ public OptionRule queryDataSourceFieldByName(String pluginName) { @Override public OptionRule queryMetadataFieldByName(String pluginName) { return getDataSourceChannel(pluginName) - .getDatasourceMetadataFieldsByDataSourceName(pluginName); + .getDatasourceMetadataFieldsByDataSourceName(pluginName); } @Override public List getTables( - String pluginName, String databaseName, Map requestParams) { + String pluginName, String databaseName, Map requestParams) { return getDataSourceChannel(pluginName).getTables(pluginName, requestParams, databaseName); } @@ -115,21 +115,21 @@ public List getDatabases(String pluginName, Map requestP @Override public List getTableFields( - String pluginName, - Map requestParams, - String databaseName, - String tableName) { + String pluginName, + Map requestParams, + String databaseName, + String tableName) { return getDataSourceChannel(pluginName) - .getTableFields(pluginName, requestParams, databaseName, tableName); + .getTableFields(pluginName, requestParams, databaseName, tableName); } @Override public Map> getTableFields( - String pluginName, - Map requestParams, - String databaseName, - List tableNames) { + String pluginName, + Map requestParams, + String databaseName, + List tableNames) { return getDataSourceChannel(pluginName) - .getTableFields(pluginName, requestParams, databaseName, tableNames); + .getTableFields(pluginName, requestParams, databaseName, tableNames); } } diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/service/DataSourceService.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/service/DataSourceService.java index 9ab18325d..5e52c0777 100644 --- a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/service/DataSourceService.java +++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/service/DataSourceService.java @@ -67,18 +67,18 @@ public interface DataSourceService { /** * get data source table names by database name * - * @param pluginName plugin name - * @param databaseName database name + * @param pluginName plugin name + * @param databaseName database name * @param requestParams connection params * @return table names */ List getTables( - String pluginName, String databaseName, Map requestParams); + String pluginName, String databaseName, Map requestParams); /** * get data source database names * - * @param pluginName plugin name + * @param pluginName plugin name * @param requestParams connection params * @return database names */ @@ -87,30 +87,30 @@ List getTables( /** * get data source table fields * - * @param pluginName plugin name + * @param pluginName plugin name * @param requestParams connection params - * @param databaseName database name - * @param tableName table name + * @param databaseName database name + * @param tableName table name * @return table fields */ List getTableFields( - String pluginName, - Map requestParams, - String databaseName, - String tableName); + String pluginName, + Map requestParams, + String databaseName, + String tableName); /** * get data source table fields * - * @param pluginName plugin name + * @param pluginName plugin name * @param requestParams connection params - * @param databaseName database name - * @param tableNames table names + * @param databaseName database name + * @param tableNames table names * @return table fields */ Map> getTableFields( - String pluginName, - Map requestParams, - String databaseName, - List tableNames); + String pluginName, + Map requestParams, + String databaseName, + List tableNames); } diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/test/java/com/whaleops/datasource/s3/S3DatasourceChannelTest.java b/seatunnel-datasource/seatunnel-datasource-client/src/test/java/com/whaleops/datasource/s3/S3DatasourceChannelTest.java index ece18206a..50c3d2d5b 100644 --- a/seatunnel-datasource/seatunnel-datasource-client/src/test/java/com/whaleops/datasource/s3/S3DatasourceChannelTest.java +++ b/seatunnel-datasource/seatunnel-datasource-client/src/test/java/com/whaleops/datasource/s3/S3DatasourceChannelTest.java @@ -19,11 +19,12 @@ import org.apache.seatunnel.datasource.plugin.s3.S3DatasourceChannel; -import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import com.google.common.collect.ImmutableMap; + import java.util.Map; @Disabled @@ -33,23 +34,23 @@ class S3DatasourceChannelTest { @Test void checkDataSourceConnectivity() { Assertions.assertDoesNotThrow( - () -> { - S3_DATASOURCE_CHANNEL.checkDataSourceConnectivity("S3", createRequestParams()); - }); + () -> { + S3_DATASOURCE_CHANNEL.checkDataSourceConnectivity("S3", createRequestParams()); + }); } private Map createRequestParams() { Map requestParams = - new ImmutableMap.Builder() - .put("bucket", "s3a://poc-kuke") - .put("fs.s3a.endpoint", "s3.cn-north-1.amazonaws.com.cn") - .put( - "fs.s3a.aws.credentials.provider", - "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider") - .put("access_key", "AKIAYYUV5DMXADXRBGTA") - .put("secret_key", "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") - .put("hadoop_s3_properties", "") - .build(); + new ImmutableMap.Builder() + .put("bucket", "s3a://poc-kuke") + .put("fs.s3a.endpoint", "s3.cn-north-1.amazonaws.com.cn") + .put( + "fs.s3a.aws.credentials.provider", + "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider") + .put("access_key", "AKIAYYUV5DMXADXRBGTA") + .put("secret_key", "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") + .put("hadoop_s3_properties", "") + .build(); return requestParams; } } diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/test/java/org/apache/seatunnel/datasource/DataSourceClientTest.java b/seatunnel-datasource/seatunnel-datasource-client/src/test/java/org/apache/seatunnel/datasource/DataSourceClientTest.java index e2877e88b..1c45fe7e1 100644 --- a/seatunnel-datasource/seatunnel-datasource-client/src/test/java/org/apache/seatunnel/datasource/DataSourceClientTest.java +++ b/seatunnel-datasource/seatunnel-datasource-client/src/test/java/org/apache/seatunnel/datasource/DataSourceClientTest.java @@ -18,6 +18,7 @@ package org.apache.seatunnel.datasource; import org.apache.commons.lang3.StringUtils; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -27,22 +28,22 @@ class DataSourceClientTest { @Test public void listAllDataSources() { Assertions.assertTrue( - DATA_SOURCE_CLIENT.listAllDataSources().stream() - .anyMatch( - dataSourcePluginInfo -> - StringUtils.equalsAnyIgnoreCase( - dataSourcePluginInfo.getName(), "jdbc-mysql"))); + DATA_SOURCE_CLIENT.listAllDataSources().stream() + .anyMatch( + dataSourcePluginInfo -> + StringUtils.equalsAnyIgnoreCase( + dataSourcePluginInfo.getName(), "jdbc-mysql"))); Assertions.assertTrue( - DATA_SOURCE_CLIENT.listAllDataSources().stream() - .anyMatch( - dataSourcePluginInfo -> - StringUtils.equalsAnyIgnoreCase( - dataSourcePluginInfo.getName(), "kafka"))); + DATA_SOURCE_CLIENT.listAllDataSources().stream() + .anyMatch( + dataSourcePluginInfo -> + StringUtils.equalsAnyIgnoreCase( + dataSourcePluginInfo.getName(), "kafka"))); Assertions.assertTrue( - DATA_SOURCE_CLIENT.listAllDataSources().stream() - .anyMatch( - dataSourcePluginInfo -> - StringUtils.equalsAnyIgnoreCase( - dataSourcePluginInfo.getName(), "elasticsearch"))); + DATA_SOURCE_CLIENT.listAllDataSources().stream() + .anyMatch( + dataSourcePluginInfo -> + StringUtils.equalsAnyIgnoreCase( + dataSourcePluginInfo.getName(), "elasticsearch"))); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannel.java index 17495487f..6a4d92436 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannel.java @@ -17,16 +17,17 @@ package org.apache.seatunnel.datasource.plugin.elasticsearch; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; + import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; import org.apache.seatunnel.datasource.plugin.elasticsearch.client.EsRestClient; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; +import org.apache.commons.lang3.StringUtils; import lombok.NonNull; -import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.HashMap; @@ -54,47 +55,47 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { databaseCheck(database); try (EsRestClient client = - EsRestClient.createInstance(ConfigFactory.parseMap(requestParams))) { + EsRestClient.createInstance(ConfigFactory.parseMap(requestParams))) { return client.listIndex(); } } @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { return DEFAULT_DATABASES; } @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try (EsRestClient client = - EsRestClient.createInstance(ConfigFactory.parseMap(requestParams))) { + EsRestClient.createInstance(ConfigFactory.parseMap(requestParams))) { client.getClusterInfo(); return true; } catch (Throwable e) { throw new DataSourcePluginException( - "check ElasticSearch connectivity failed, " + e.getMessage(), e); + "check ElasticSearch connectivity failed, " + e.getMessage(), e); } } @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { databaseCheck(database); try (EsRestClient client = - EsRestClient.createInstance(ConfigFactory.parseMap(requestParams))) { + EsRestClient.createInstance(ConfigFactory.parseMap(requestParams))) { Map fieldTypeMapping = client.getFieldTypeMapping(table); List fields = new ArrayList<>(); fieldTypeMapping.forEach( - (fieldName, fieldType) -> - fields.add(convertToTableField(fieldName, fieldType))); + (fieldName, fieldType) -> + fields.add(convertToTableField(fieldName, fieldType))); return fields; } catch (Exception ex) { throw new DataSourcePluginException("Get table fields failed", ex); @@ -103,16 +104,16 @@ public List getTableFields( @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { databaseCheck(database); Map> tableFields = new HashMap<>(); tables.forEach( - table -> - tableFields.put( - table, getTableFields(pluginName, requestParams, database, table))); + table -> + tableFields.put( + table, getTableFields(pluginName, requestParams, database, table))); return tableFields; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/EsRestClient.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/EsRestClient.java index 1ef2e7050..4de06bf4a 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/EsRestClient.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/EsRestClient.java @@ -17,15 +17,14 @@ package org.apache.seatunnel.datasource.plugin.elasticsearch.client; -import org.apache.seatunnel.common.utils.JsonUtils; -import org.apache.seatunnel.datasource.plugin.elasticsearch.ElasticSearchOptionRule; - import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.seatunnel.shade.com.typesafe.config.Config; -import lombok.extern.slf4j.Slf4j; +import org.apache.seatunnel.common.utils.JsonUtils; +import org.apache.seatunnel.datasource.plugin.elasticsearch.ElasticSearchOptionRule; + import org.apache.http.HttpHost; import org.apache.http.HttpStatus; import org.apache.http.auth.AuthScope; @@ -36,11 +35,14 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; + import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; +import lombok.extern.slf4j.Slf4j; + import javax.net.ssl.SSLContext; import java.io.IOException; @@ -69,18 +71,18 @@ private EsRestClient(RestClient restClient) { public static EsRestClient createInstance(Config pluginConfig) { try { List hosts = - OBJECT_MAPPER.readValue( - pluginConfig.getString(ElasticSearchOptionRule.HOSTS.key()), - List.class); + OBJECT_MAPPER.readValue( + pluginConfig.getString(ElasticSearchOptionRule.HOSTS.key()), + List.class); Optional username = Optional.empty(); Optional password = Optional.empty(); if (pluginConfig.hasPath(ElasticSearchOptionRule.USERNAME.key())) { username = - Optional.of(pluginConfig.getString(ElasticSearchOptionRule.USERNAME.key())); + Optional.of(pluginConfig.getString(ElasticSearchOptionRule.USERNAME.key())); if (pluginConfig.hasPath(ElasticSearchOptionRule.PASSWORD.key())) { password = - Optional.of( - pluginConfig.getString(ElasticSearchOptionRule.PASSWORD.key())); + Optional.of( + pluginConfig.getString(ElasticSearchOptionRule.PASSWORD.key())); } } Optional keystorePath = Optional.empty(); @@ -88,141 +90,141 @@ public static EsRestClient createInstance(Config pluginConfig) { Optional truststorePath = Optional.empty(); Optional truststorePassword = Optional.empty(); boolean tlsVerifyCertificate = - ElasticSearchOptionRule.TLS_VERIFY_CERTIFICATE.defaultValue(); + ElasticSearchOptionRule.TLS_VERIFY_CERTIFICATE.defaultValue(); if (pluginConfig.hasPath(ElasticSearchOptionRule.TLS_VERIFY_CERTIFICATE.key())) { tlsVerifyCertificate = - pluginConfig.getBoolean( - ElasticSearchOptionRule.TLS_VERIFY_CERTIFICATE.key()); + pluginConfig.getBoolean( + ElasticSearchOptionRule.TLS_VERIFY_CERTIFICATE.key()); } if (tlsVerifyCertificate) { if (pluginConfig.hasPath(ElasticSearchOptionRule.TLS_KEY_STORE_PATH.key())) { keystorePath = - Optional.of( - pluginConfig.getString( - ElasticSearchOptionRule.TLS_KEY_STORE_PATH.key())); + Optional.of( + pluginConfig.getString( + ElasticSearchOptionRule.TLS_KEY_STORE_PATH.key())); } if (pluginConfig.hasPath(ElasticSearchOptionRule.TLS_KEY_STORE_PASSWORD.key())) { keystorePassword = - Optional.of( - pluginConfig.getString( - ElasticSearchOptionRule.TLS_KEY_STORE_PASSWORD.key())); + Optional.of( + pluginConfig.getString( + ElasticSearchOptionRule.TLS_KEY_STORE_PASSWORD.key())); } if (pluginConfig.hasPath(ElasticSearchOptionRule.TLS_TRUST_STORE_PATH.key())) { truststorePath = - Optional.of( - pluginConfig.getString( - ElasticSearchOptionRule.TLS_TRUST_STORE_PATH.key())); + Optional.of( + pluginConfig.getString( + ElasticSearchOptionRule.TLS_TRUST_STORE_PATH.key())); } if (pluginConfig.hasPath(ElasticSearchOptionRule.TLS_TRUST_STORE_PASSWORD.key())) { truststorePassword = - Optional.of( - pluginConfig.getString( - ElasticSearchOptionRule.TLS_TRUST_STORE_PASSWORD - .key())); + Optional.of( + pluginConfig.getString( + ElasticSearchOptionRule.TLS_TRUST_STORE_PASSWORD + .key())); } } boolean tlsVerifyHostnames = ElasticSearchOptionRule.TLS_VERIFY_HOSTNAME.defaultValue(); if (pluginConfig.hasPath(ElasticSearchOptionRule.TLS_VERIFY_HOSTNAME.key())) { tlsVerifyHostnames = - pluginConfig.getBoolean(ElasticSearchOptionRule.TLS_VERIFY_HOSTNAME.key()); + pluginConfig.getBoolean(ElasticSearchOptionRule.TLS_VERIFY_HOSTNAME.key()); } return createInstance( - hosts, - username, - password, - tlsVerifyCertificate, - tlsVerifyHostnames, - keystorePath, - keystorePassword, - truststorePath, - truststorePassword); + hosts, + username, + password, + tlsVerifyCertificate, + tlsVerifyHostnames, + keystorePath, + keystorePassword, + truststorePath, + truststorePassword); } catch (Exception e) { throw new RuntimeException("Create EsRestClient failed", e); } } public static EsRestClient createInstance( - List hosts, - Optional username, - Optional password, - boolean tlsVerifyCertificate, - boolean tlsVerifyHostnames, - Optional keystorePath, - Optional keystorePassword, - Optional truststorePath, - Optional truststorePassword) { + List hosts, + Optional username, + Optional password, + boolean tlsVerifyCertificate, + boolean tlsVerifyHostnames, + Optional keystorePath, + Optional keystorePassword, + Optional truststorePath, + Optional truststorePassword) { RestClientBuilder restClientBuilder = - getRestClientBuilder( - hosts, - username, - password, - tlsVerifyCertificate, - tlsVerifyHostnames, - keystorePath, - keystorePassword, - truststorePath, - truststorePassword); + getRestClientBuilder( + hosts, + username, + password, + tlsVerifyCertificate, + tlsVerifyHostnames, + keystorePath, + keystorePassword, + truststorePath, + truststorePassword); return new EsRestClient(restClientBuilder.build()); } private static RestClientBuilder getRestClientBuilder( - List hosts, - Optional username, - Optional password, - boolean tlsVerifyCertificate, - boolean tlsVerifyHostnames, - Optional keystorePath, - Optional keystorePassword, - Optional truststorePath, - Optional truststorePassword) { + List hosts, + Optional username, + Optional password, + boolean tlsVerifyCertificate, + boolean tlsVerifyHostnames, + Optional keystorePath, + Optional keystorePassword, + Optional truststorePath, + Optional truststorePassword) { HttpHost[] httpHosts = new HttpHost[hosts.size()]; for (int i = 0; i < hosts.size(); i++) { httpHosts[i] = HttpHost.create(hosts.get(i)); } RestClientBuilder restClientBuilder = - RestClient.builder(httpHosts) - .setRequestConfigCallback( - requestConfigBuilder -> - requestConfigBuilder - .setConnectionRequestTimeout( - CONNECTION_REQUEST_TIMEOUT) - .setSocketTimeout(SOCKET_TIMEOUT)); + RestClient.builder(httpHosts) + .setRequestConfigCallback( + requestConfigBuilder -> + requestConfigBuilder + .setConnectionRequestTimeout( + CONNECTION_REQUEST_TIMEOUT) + .setSocketTimeout(SOCKET_TIMEOUT)); restClientBuilder.setHttpClientConfigCallback( - httpClientBuilder -> { - if (username.isPresent()) { - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials( - AuthScope.ANY, - new UsernamePasswordCredentials(username.get(), password.get())); - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); - } - - try { - if (tlsVerifyCertificate) { - Optional sslContext = - SSLUtils.buildSSLContext( - keystorePath, - keystorePassword, - truststorePath, - truststorePassword); - sslContext.ifPresent(e -> httpClientBuilder.setSSLContext(e)); - } else { - SSLContext sslContext = - SSLContexts.custom() - .loadTrustMaterial(new TrustAllStrategy()) - .build(); - httpClientBuilder.setSSLContext(sslContext); + httpClientBuilder -> { + if (username.isPresent()) { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials(username.get(), password.get())); + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); } - if (!tlsVerifyHostnames) { - httpClientBuilder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); + + try { + if (tlsVerifyCertificate) { + Optional sslContext = + SSLUtils.buildSSLContext( + keystorePath, + keystorePassword, + truststorePath, + truststorePassword); + sslContext.ifPresent(e -> httpClientBuilder.setSSLContext(e)); + } else { + SSLContext sslContext = + SSLContexts.custom() + .loadTrustMaterial(new TrustAllStrategy()) + .build(); + httpClientBuilder.setSSLContext(sslContext); + } + if (!tlsVerifyHostnames) { + httpClientBuilder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); + } + } catch (Exception e) { + throw new RuntimeException(e); } - } catch (Exception e) { - throw new RuntimeException(e); - } - return httpClientBuilder; - }); + return httpClientBuilder; + }); return restClientBuilder; } @@ -235,12 +237,12 @@ public ElasticsearchClusterInfo getClusterInfo() { JsonNode jsonNode = objectMapper.readTree(result); JsonNode versionNode = jsonNode.get("version"); return ElasticsearchClusterInfo.builder() - .clusterVersion(versionNode.get("number").asText()) - .distribution( - Optional.ofNullable(versionNode.get("distribution")) - .map(JsonNode::asText) - .orElse(null)) - .build(); + .clusterVersion(versionNode.get("number").asText()) + .distribution( + Optional.ofNullable(versionNode.get("distribution")) + .map(JsonNode::asText) + .orElse(null)) + .build(); } catch (IOException e) { throw new ResponseException("fail to get elasticsearch version.", e); } @@ -265,13 +267,13 @@ public List listIndex() { if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { String entity = EntityUtils.toString(response.getEntity()); return JsonUtils.toList(entity, Map.class).stream() - .map(map -> map.get("index").toString()) - .collect(Collectors.toList()); + .map(map -> map.get("index").toString()) + .collect(Collectors.toList()); } else { throw new ResponseException( - String.format( - "GET %s response status code=%d", - endpoint, response.getStatusLine().getStatusCode())); + String.format( + "GET %s response status code=%d", + endpoint, response.getStatusLine().getStatusCode())); } } catch (IOException ex) { throw new ResponseException(ex); @@ -289,9 +291,9 @@ public void dropIndex(String tableName) { // todo: if the index doesn't exist, the response status code is 200? if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { throw new ResponseException( - String.format( - "DELETE %s response status code=%d", - endpoint, response.getStatusLine().getStatusCode())); + String.format( + "DELETE %s response status code=%d", + endpoint, response.getStatusLine().getStatusCode())); } } catch (IOException ex) { throw new ResponseException(ex); @@ -315,9 +317,9 @@ public Map getFieldTypeMapping(String index) { } if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { throw new ResponseException( - String.format( - "GET %s response status code=%d", - endpoint, response.getStatusLine().getStatusCode())); + String.format( + "GET %s response status code=%d", + endpoint, response.getStatusLine().getStatusCode())); } String entity = EntityUtils.toString(response.getEntity()); log.info(String.format("GET %s response=%s", endpoint, entity)); @@ -359,9 +361,9 @@ private static Map getFieldTypeMappingFromProperties(JsonNode pr mapping.put(field, type); } else { log.warn( - String.format( - "fail to get elasticsearch field %s mapping type,so give a default type text", - field)); + String.format( + "fail to get elasticsearch field %s mapping type,so give a default type text", + field)); mapping.put(field, "text"); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/SSLUtils.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/SSLUtils.java index 9c10cd3b8..8f3caa204 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/SSLUtils.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/SSLUtils.java @@ -17,8 +17,6 @@ package org.apache.seatunnel.datasource.plugin.elasticsearch.client; -import static java.util.Collections.list; - import io.airlift.security.pem.PemReader; import javax.net.ssl.KeyManager; @@ -43,29 +41,31 @@ import java.util.List; import java.util.Optional; +import static java.util.Collections.list; + @SuppressWarnings("MagicNumber") public final class SSLUtils { public static Optional buildSSLContext( - Optional keyStorePath, - Optional keyStorePassword, - Optional trustStorePath, - Optional trustStorePassword) - throws GeneralSecurityException, IOException { + Optional keyStorePath, + Optional keyStorePassword, + Optional trustStorePath, + Optional trustStorePassword) + throws GeneralSecurityException, IOException { if (!keyStorePath.isPresent() && !trustStorePath.isPresent()) { return Optional.empty(); } return Optional.of( - createSSLContext( - keyStorePath, keyStorePassword, trustStorePath, trustStorePassword)); + createSSLContext( + keyStorePath, keyStorePassword, trustStorePath, trustStorePassword)); } private static SSLContext createSSLContext( - Optional keyStorePath, - Optional keyStorePassword, - Optional trustStorePath, - Optional trustStorePassword) - throws GeneralSecurityException, IOException { + Optional keyStorePath, + Optional keyStorePassword, + Optional trustStorePath, + Optional trustStorePassword) + throws GeneralSecurityException, IOException { // load KeyStore if configured and get KeyManagers KeyStore keyStore = null; KeyManager[] keyManagers = null; @@ -88,7 +88,7 @@ private static SSLContext createSSLContext( } validateCertificates(keyStore); KeyManagerFactory keyManagerFactory = - KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keyManagerFactory.init(keyStore, keyManagerPassword); keyManagers = keyManagerFactory.getKeyManagers(); } @@ -102,14 +102,14 @@ private static SSLContext createSSLContext( // create TrustManagerFactory TrustManagerFactory trustManagerFactory = - TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); trustManagerFactory.init(trustStore); // get X509TrustManager TrustManager[] trustManagers = trustManagerFactory.getTrustManagers(); if (trustManagers.length != 1 || !(trustManagers[0] instanceof X509TrustManager)) { throw new RuntimeException( - "Unexpected default trust managers:" + Arrays.toString(trustManagers)); + "Unexpected default trust managers:" + Arrays.toString(trustManagers)); } // create SSLContext SSLContext result = SSLContext.getInstance("SSL"); @@ -118,7 +118,7 @@ private static SSLContext createSSLContext( } private static KeyStore loadTrustStore(File trustStorePath, Optional trustStorePassword) - throws IOException, GeneralSecurityException { + throws IOException, GeneralSecurityException { KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); try { // attempt to read the trust store as a PEM file @@ -155,10 +155,10 @@ private static void validateCertificates(KeyStore keyStore) throws GeneralSecuri ((X509Certificate) certificate).checkValidity(); } catch (CertificateExpiredException e) { throw new CertificateExpiredException( - "KeyStore certificate is expired: " + e.getMessage()); + "KeyStore certificate is expired: " + e.getMessage()); } catch (CertificateNotYetValidException e) { throw new CertificateNotYetValidException( - "KeyStore certificate is not yet valid: " + e.getMessage()); + "KeyStore certificate is not yet valid: " + e.getMessage()); } } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannelTest.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannelTest.java index b8b25d841..b888a1ff7 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannelTest.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannelTest.java @@ -19,14 +19,15 @@ import org.apache.seatunnel.datasource.plugin.api.model.TableField; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; + import java.util.List; import java.util.Map; @@ -34,19 +35,19 @@ @Disabled class ElasticSearchDataSourceChannelTest { private static final Logger LOGGER = - LoggerFactory.getLogger(ElasticSearchDataSourceChannelTest.class); + LoggerFactory.getLogger(ElasticSearchDataSourceChannelTest.class); private static final ElasticSearchDataSourceChannel ELASTIC_SEARCH_DATA_SOURCE_CHANNEL = - new ElasticSearchDataSourceChannel(); + new ElasticSearchDataSourceChannel(); private static final String PLUGIN_NAME = "ElasticSearch"; private static final String DATABASE = "Default"; private static final Map REQUEST_MAP = - new ImmutableMap.Builder() - .put(ElasticSearchOptionRule.HOSTS.key(), "[\"http://localhost:9200\"]") - .build(); + new ImmutableMap.Builder() + .put(ElasticSearchOptionRule.HOSTS.key(), "[\"http://localhost:9200\"]") + .build(); @Test void canAbleGetSchema() { @@ -55,60 +56,61 @@ void canAbleGetSchema() { @Test void getDataSourceOptions() { - Assertions.assertNotNull(ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getDataSourceOptions(PLUGIN_NAME)); + Assertions.assertNotNull( + ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getDataSourceOptions(PLUGIN_NAME)); } @Test void getDatasourceMetadataFieldsByDataSourceName() { Assertions.assertNotNull( - ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getDatasourceMetadataFieldsByDataSourceName( - PLUGIN_NAME)); + ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getDatasourceMetadataFieldsByDataSourceName( + PLUGIN_NAME)); } @Test void getTables() { Assertions.assertDoesNotThrow( - () -> { - List tables = - ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getTables( - PLUGIN_NAME, REQUEST_MAP, DATABASE); - LOGGER.info("{}", tables); - }); + () -> { + List tables = + ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getTables( + PLUGIN_NAME, REQUEST_MAP, DATABASE); + LOGGER.info("{}", tables); + }); } @Test void getDatabases() { Assertions.assertLinesMatch( - Lists.newArrayList("default"), - ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getDatabases(PLUGIN_NAME, REQUEST_MAP)); + Lists.newArrayList("default"), + ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getDatabases(PLUGIN_NAME, REQUEST_MAP)); } @Test void checkDataSourceConnectivity() { Assertions.assertTrue( - ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.checkDataSourceConnectivity( - PLUGIN_NAME, REQUEST_MAP)); + ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.checkDataSourceConnectivity( + PLUGIN_NAME, REQUEST_MAP)); } @Test void getTableFields() { Assertions.assertDoesNotThrow( - () -> { - List tableFields = - ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getTableFields( - PLUGIN_NAME, REQUEST_MAP, DATABASE, ""); - LOGGER.info("{}", tableFields); - }); + () -> { + List tableFields = + ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getTableFields( + PLUGIN_NAME, REQUEST_MAP, DATABASE, ""); + LOGGER.info("{}", tableFields); + }); } @Test void testGetTableFields() { Assertions.assertDoesNotThrow( - () -> { - Map> tableFields = - ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getTableFields( - PLUGIN_NAME, REQUEST_MAP, DATABASE, Lists.newArrayList("")); - LOGGER.info("{}", tableFields); - }); + () -> { + Map> tableFields = + ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getTableFields( + PLUGIN_NAME, REQUEST_MAP, DATABASE, Lists.newArrayList("")); + LOGGER.info("{}", tableFields); + }); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceFactoryTest.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceFactoryTest.java index 85a49e77b..0441d7a12 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceFactoryTest.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceFactoryTest.java @@ -23,12 +23,12 @@ class ElasticSearchDataSourceFactoryTest { private static final ElasticSearchDataSourceFactory ELASTIC_SEARCH_DATA_SOURCE_FACTORY = - new ElasticSearchDataSourceFactory(); + new ElasticSearchDataSourceFactory(); @Test void factoryIdentifier() { Assertions.assertEquals( - "ElasticSearch", ELASTIC_SEARCH_DATA_SOURCE_FACTORY.factoryIdentifier()); + "ElasticSearch", ELASTIC_SEARCH_DATA_SOURCE_FACTORY.factoryIdentifier()); } @Test diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseDataSourceConfig.java index 4862d41d8..4c5b3b2e7 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseDataSourceConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseDataSourceConfig.java @@ -29,31 +29,31 @@ public class ClickhouseDataSourceConfig { public static final String PLUGIN_NAME = "JDBC-ClickHouse"; public static final DataSourcePluginInfo CLICKHOUSE_DATASOURCE_PLUGIN_INFO = - DataSourcePluginInfo.builder() - .name(PLUGIN_NAME) - .icon(PLUGIN_NAME) - .version("1.0.0") - .type(DatasourcePluginTypeEnum.DATABASE.getCode()) - .supportVirtualTables(false) - .build(); + DataSourcePluginInfo.builder() + .name(PLUGIN_NAME) + .icon(PLUGIN_NAME) + .version("1.0.0") + .type(DatasourcePluginTypeEnum.DATABASE.getCode()) + .supportVirtualTables(false) + .build(); public static final Set CLICKHOUSE_SYSTEM_DATABASES = - Sets.newHashSet( - "system", - "default", - "information_schema", - "mysql", - "performance_schema", - "sys"); + Sets.newHashSet( + "system", + "default", + "information_schema", + "mysql", + "performance_schema", + "sys"); public static final OptionRule OPTION_RULE = - OptionRule.builder() - .required(ClickhouseOptionRule.URL, ClickhouseOptionRule.DRIVER) - .optional(ClickhouseOptionRule.USER, ClickhouseOptionRule.PASSWORD) - .build(); + OptionRule.builder() + .required(ClickhouseOptionRule.URL, ClickhouseOptionRule.DRIVER) + .optional(ClickhouseOptionRule.USER, ClickhouseOptionRule.PASSWORD) + .build(); public static final OptionRule METADATA_RULE = - OptionRule.builder() - .required(ClickhouseOptionRule.DATABASE, ClickhouseOptionRule.TABLE) - .build(); + OptionRule.builder() + .required(ClickhouseOptionRule.DATABASE, ClickhouseOptionRule.TABLE) + .build(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceChannel.java index 371411741..ef160afbc 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceChannel.java @@ -17,17 +17,16 @@ package org.apache.seatunnel.datasource.plugin.clickhouse.jdbc; -import static com.google.common.base.Preconditions.checkNotNull; - import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; +import org.apache.commons.lang3.StringUtils; + import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -39,6 +38,8 @@ import java.util.List; import java.util.Map; +import static com.google.common.base.Preconditions.checkNotNull; + @Slf4j public class ClickhouseJdbcDataSourceChannel implements DataSourceChannel { @@ -54,13 +55,13 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { List tableNames = new ArrayList<>(); - try (Connection connection = getConnection(requestParams);) { + try (Connection connection = getConnection(requestParams); ) { ResultSet resultSet = - connection - .getMetaData() - .getTables(database, null, null, new String[]{"TABLE"}); + connection + .getMetaData() + .getTables(database, null, null, new String[] {"TABLE"}); while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); if (StringUtils.isNotBlank(tableName)) { @@ -75,17 +76,17 @@ public List getTables( @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { List dbNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - Statement statement = connection.createStatement(); - ResultSet re = statement.executeQuery("SHOW DATABASES;")) { + Statement statement = connection.createStatement(); + ResultSet re = statement.executeQuery("SHOW DATABASES;")) { // filter system databases while (re.next()) { String dbName = re.getString("name"); if (StringUtils.isNotBlank(dbName) - && !ClickhouseDataSourceConfig.CLICKHOUSE_SYSTEM_DATABASES.contains( - dbName)) { + && !ClickhouseDataSourceConfig.CLICKHOUSE_SYSTEM_DATABASES.contains( + dbName)) { dbNames.add(dbName); } } @@ -97,7 +98,7 @@ public List getDatabases( @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try (Connection ignored = getConnection(requestParams)) { return true; } catch (Exception e) { @@ -107,10 +108,10 @@ public boolean checkDataSourceConnectivity( @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { List tableFields = new ArrayList<>(); try (Connection connection = getConnection(requestParams, database)) { DatabaseMetaData metaData = connection.getMetaData(); @@ -139,15 +140,15 @@ public List getTableFields( @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { return null; } private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { + throws SQLException { ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); while (primaryKeysInfo.next()) { return primaryKeysInfo.getString("COLUMN_NAME"); @@ -156,17 +157,17 @@ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String ta } private Connection getConnection(Map requestParams) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { return getConnection(requestParams, null); } private Connection getConnection(Map requestParams, String databaseName) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { checkNotNull(requestParams.get(ClickhouseOptionRule.DRIVER.key())); checkNotNull(requestParams.get(ClickhouseOptionRule.URL.key()), "Jdbc url cannot be null"); String url = - JdbcUtils.replaceDatabase( - requestParams.get(ClickhouseOptionRule.URL.key()), databaseName); + JdbcUtils.replaceDatabase( + requestParams.get(ClickhouseOptionRule.URL.key()), databaseName); if (requestParams.containsKey(ClickhouseOptionRule.USER.key())) { String username = requestParams.get(ClickhouseOptionRule.USER.key()); String password = requestParams.get(ClickhouseOptionRule.PASSWORD.key()); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseOptionRule.java index 3beff6058..f02e6030d 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseOptionRule.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseOptionRule.java @@ -23,30 +23,30 @@ public class ClickhouseOptionRule { public static final Option URL = - Options.key("url") - .stringType() - .noDefaultValue() - .withDescription( - "jdbc url, eg:" - + "jdbc:clickhouse://localhost:8123/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); + Options.key("url") + .stringType() + .noDefaultValue() + .withDescription( + "jdbc url, eg:" + + "jdbc:clickhouse://localhost:8123/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); public static final Option USER = - Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); + Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); public static final Option PASSWORD = - Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); + Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); public static final Option DATABASE = - Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); + Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); public static final Option TABLE = - Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); + Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); public static final Option DRIVER = - Options.key("driver") - .enumType(DriverType.class) - .noDefaultValue() - .withDescription("driver"); + Options.key("driver") + .enumType(DriverType.class) + .noDefaultValue() + .withDescription("driver"); public enum DriverType { ClickHouse("ru.yandex.clickhouse.ClickHouseDriver"); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceChannel.java index 32543c423..62559037d 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceChannel.java @@ -22,11 +22,12 @@ import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; @@ -56,13 +57,13 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { return getTables(pluginName, requestParams, database); } @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try { return getDataBaseNames(pluginName, requestParams); } catch (SQLException e) { @@ -73,25 +74,25 @@ public List getDatabases( @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { return checkJdbcConnectivity(requestParams); } @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { return getTableFields(requestParams, database, table); } @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { Map> tableFields = new HashMap<>(tables.size()); for (String table : tables) { tableFields.put(table, getTableFields(requestParams, database, table)); @@ -104,24 +105,24 @@ protected boolean checkJdbcConnectivity(Map requestParams) { return true; } catch (Exception e) { throw new DataSourcePluginException( - "check jdbc connectivity failed, " + e.getMessage(), e); + "check jdbc connectivity failed, " + e.getMessage(), e); } } protected Connection init(Map requestParams) throws SQLException { if (MapUtils.isEmpty(requestParams)) { throw new DataSourcePluginException( - "Hive jdbc request params is null, please check your config"); + "Hive jdbc request params is null, please check your config"); } String url = requestParams.get(HiveJdbcOptionRule.URL.key()); return DriverManager.getConnection(url); } protected List getDataBaseNames(String pluginName, Map requestParams) - throws SQLException { + throws SQLException { List dbNames = new ArrayList<>(); try (Connection connection = init(requestParams); - Statement statement = connection.createStatement();) { + Statement statement = connection.createStatement(); ) { ResultSet re = statement.executeQuery("SHOW DATABASES;"); // filter system databases while (re.next()) { @@ -136,9 +137,9 @@ protected List getDataBaseNames(String pluginName, Map r protected List getTableNames(Map requestParams, String dbName) { List tableNames = new ArrayList<>(); - try (Connection connection = init(requestParams);) { + try (Connection connection = init(requestParams); ) { ResultSet resultSet = - connection.getMetaData().getTables(dbName, null, null, new String[]{"TABLE"}); + connection.getMetaData().getTables(dbName, null, null, new String[] {"TABLE"}); while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); if (StringUtils.isNotBlank(tableName)) { @@ -152,9 +153,9 @@ protected List getTableNames(Map requestParams, String d } protected List getTableFields( - Map requestParams, String dbName, String tableName) { + Map requestParams, String dbName, String tableName) { List tableFields = new ArrayList<>(); - try (Connection connection = init(requestParams);) { + try (Connection connection = init(requestParams); ) { DatabaseMetaData metaData = connection.getMetaData(); String primaryKey = getPrimaryKey(metaData, dbName, tableName); ResultSet resultSet = metaData.getColumns(dbName, null, tableName, null); @@ -180,7 +181,7 @@ protected List getTableFields( } private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { + throws SQLException { ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); while (primaryKeysInfo.next()) { return primaryKeysInfo.getString("COLUMN_NAME"); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlDataSourceConfig.java index 2e2de3e45..03d6d0cf3 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlDataSourceConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlDataSourceConfig.java @@ -30,22 +30,22 @@ public class MysqlDataSourceConfig { public static final String PLUGIN_NAME = "JDBC-Mysql"; public static final DataSourcePluginInfo MYSQL_DATASOURCE_PLUGIN_INFO = - DataSourcePluginInfo.builder() - .name(PLUGIN_NAME) - .icon(PLUGIN_NAME) - .version("1.0.0") - .type(DatasourcePluginTypeEnum.DATABASE.getCode()) - .build(); + DataSourcePluginInfo.builder() + .name(PLUGIN_NAME) + .icon(PLUGIN_NAME) + .version("1.0.0") + .type(DatasourcePluginTypeEnum.DATABASE.getCode()) + .build(); public static final Set MYSQL_SYSTEM_DATABASES = - Sets.newHashSet("information_schema", "mysql", "performance_schema", "sys"); + Sets.newHashSet("information_schema", "mysql", "performance_schema", "sys"); public static final OptionRule OPTION_RULE = - OptionRule.builder() - .required(MysqlOptionRule.URL, MysqlOptionRule.DRIVER) - .optional(MysqlOptionRule.USER, MysqlOptionRule.PASSWORD) - .build(); + OptionRule.builder() + .required(MysqlOptionRule.URL, MysqlOptionRule.DRIVER) + .optional(MysqlOptionRule.USER, MysqlOptionRule.PASSWORD) + .build(); public static final OptionRule METADATA_RULE = - OptionRule.builder().required(MysqlOptionRule.DATABASE, MysqlOptionRule.TABLE).build(); + OptionRule.builder().required(MysqlOptionRule.DATABASE, MysqlOptionRule.TABLE).build(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java index 1fe420d53..99ab042b9 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java @@ -17,17 +17,16 @@ package org.apache.seatunnel.datasource.plugin.mysql.jdbc; -import static com.google.common.base.Preconditions.checkNotNull; - import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; -import lombok.NonNull; import org.apache.commons.lang3.StringUtils; +import lombok.NonNull; + import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.DriverManager; @@ -40,6 +39,8 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static com.google.common.base.Preconditions.checkNotNull; + public class MysqlJdbcDataSourceChannel implements DataSourceChannel { @Override @@ -54,13 +55,13 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { List tableNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - ResultSet resultSet = - connection - .getMetaData() - .getTables(database, null, null, new String[]{"TABLE"})) { + ResultSet resultSet = + connection + .getMetaData() + .getTables(database, null, null, new String[] {"TABLE"})) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); if (StringUtils.isNotBlank(tableName)) { @@ -75,16 +76,16 @@ public List getTables( @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { List dbNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); - ResultSet re = statement.executeQuery()) { + PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); + ResultSet re = statement.executeQuery()) { // filter system databases while (re.next()) { String dbName = re.getString("database"); if (StringUtils.isNotBlank(dbName) - && !MysqlDataSourceConfig.MYSQL_SYSTEM_DATABASES.contains(dbName)) { + && !MysqlDataSourceConfig.MYSQL_SYSTEM_DATABASES.contains(dbName)) { dbNames.add(dbName); } } @@ -96,7 +97,7 @@ public List getDatabases( @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try (Connection ignored = getConnection(requestParams)) { return true; } catch (Exception e) { @@ -106,10 +107,10 @@ public boolean checkDataSourceConnectivity( @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { List tableFields = new ArrayList<>(); try (Connection connection = getConnection(requestParams, database)) { DatabaseMetaData metaData = connection.getMetaData(); @@ -138,21 +139,21 @@ public List getTableFields( @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { return tables.parallelStream() - .collect( - Collectors.toMap( - Function.identity(), - table -> - getTableFields( - pluginName, requestParams, database, table))); + .collect( + Collectors.toMap( + Function.identity(), + table -> + getTableFields( + pluginName, requestParams, database, table))); } private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { + throws SQLException { ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); while (primaryKeysInfo.next()) { return primaryKeysInfo.getString("COLUMN_NAME"); @@ -161,17 +162,17 @@ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String ta } private Connection getConnection(Map requestParams) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { return getConnection(requestParams, null); } private Connection getConnection(Map requestParams, String databaseName) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { checkNotNull(requestParams.get(MysqlOptionRule.DRIVER.key())); checkNotNull(requestParams.get(MysqlOptionRule.URL.key()), "Jdbc url cannot be null"); String url = - JdbcUtils.replaceDatabase( - requestParams.get(MysqlOptionRule.URL.key()), databaseName); + JdbcUtils.replaceDatabase( + requestParams.get(MysqlOptionRule.URL.key()), databaseName); if (requestParams.containsKey(MysqlOptionRule.USER.key())) { String username = requestParams.get(MysqlOptionRule.USER.key()); String password = requestParams.get(MysqlOptionRule.PASSWORD.key()); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlOptionRule.java index f666251fc..e34696410 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlOptionRule.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlOptionRule.java @@ -23,30 +23,30 @@ public class MysqlOptionRule { public static final Option URL = - Options.key("url") - .stringType() - .noDefaultValue() - .withDescription( - "jdbc url, eg:" - + " jdbc:mysql://localhost:3306/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); + Options.key("url") + .stringType() + .noDefaultValue() + .withDescription( + "jdbc url, eg:" + + " jdbc:mysql://localhost:3306/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); public static final Option USER = - Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); + Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); public static final Option PASSWORD = - Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); + Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); public static final Option DATABASE = - Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); + Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); public static final Option TABLE = - Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); + Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); public static final Option DRIVER = - Options.key("driver") - .enumType(DriverType.class) - .defaultValue(DriverType.MYSQL) - .withDescription("driver"); + Options.key("driver") + .enumType(DriverType.class) + .defaultValue(DriverType.MYSQL) + .withDescription("driver"); public enum DriverType { MYSQL("com.mysql.cj.jdbc.Driver"), diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java index 91ccc6615..87a04fcf6 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java @@ -17,17 +17,16 @@ package org.apache.seatunnel.datasource.plugin.oracle.jdbc; -import static com.google.common.base.Preconditions.checkNotNull; - import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; -import lombok.NonNull; import org.apache.commons.lang3.StringUtils; +import lombok.NonNull; + import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.DriverManager; @@ -38,6 +37,8 @@ import java.util.List; import java.util.Map; +import static com.google.common.base.Preconditions.checkNotNull; + public class OracleDataSourceChannel implements DataSourceChannel { @Override @@ -52,13 +53,13 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { List tableNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - ResultSet resultSet = - connection - .getMetaData() - .getTables(database, null, null, new String[]{"TABLE"});) { + ResultSet resultSet = + connection + .getMetaData() + .getTables(database, null, null, new String[] {"TABLE"}); ) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); if (StringUtils.isNotBlank(tableName)) { @@ -73,16 +74,16 @@ public List getTables( @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { List dbNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); - ResultSet re = statement.executeQuery()) { + PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); + ResultSet re = statement.executeQuery()) { // filter system databases while (re.next()) { String dbName = re.getString("database"); if (StringUtils.isNotBlank(dbName) - && !OracleDataSourceConfig.ORACLE_SYSTEM_DATABASES.contains(dbName)) { + && !OracleDataSourceConfig.ORACLE_SYSTEM_DATABASES.contains(dbName)) { dbNames.add(dbName); } } @@ -94,7 +95,7 @@ public List getDatabases( @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try (Connection ignored = getConnection(requestParams)) { return true; } catch (Exception e) { @@ -104,10 +105,10 @@ public boolean checkDataSourceConnectivity( @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { List tableFields = new ArrayList<>(); try (Connection connection = getConnection(requestParams, database)) { DatabaseMetaData metaData = connection.getMetaData(); @@ -136,15 +137,15 @@ public List getTableFields( @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { return null; } private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { + throws SQLException { ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); while (primaryKeysInfo.next()) { return primaryKeysInfo.getString("COLUMN_NAME"); @@ -153,17 +154,17 @@ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String ta } private Connection getConnection(Map requestParams) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { return getConnection(requestParams, null); } private Connection getConnection(Map requestParams, String databaseName) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { checkNotNull(requestParams.get(OracleOptionRule.DRIVER.key())); checkNotNull(requestParams.get(OracleOptionRule.URL.key()), "Jdbc url cannot be null"); String url = - JdbcUtils.replaceDatabase( - requestParams.get(OracleOptionRule.URL.key()), databaseName); + JdbcUtils.replaceDatabase( + requestParams.get(OracleOptionRule.URL.key()), databaseName); if (requestParams.containsKey(OracleOptionRule.USER.key())) { String username = requestParams.get(OracleOptionRule.USER.key()); String password = requestParams.get(OracleOptionRule.PASSWORD.key()); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceConfig.java index 96f09fbef..83455b6d7 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceConfig.java @@ -30,24 +30,24 @@ public class OracleDataSourceConfig { public static final String PLUGIN_NAME = "JDBC-Oracle"; public static final DataSourcePluginInfo ORACLE_DATASOURCE_PLUGIN_INFO = - DataSourcePluginInfo.builder() - .name(PLUGIN_NAME) - .icon(PLUGIN_NAME) - .version("1.0.0") - .type(DatasourcePluginTypeEnum.DATABASE.getCode()) - .build(); + DataSourcePluginInfo.builder() + .name(PLUGIN_NAME) + .icon(PLUGIN_NAME) + .version("1.0.0") + .type(DatasourcePluginTypeEnum.DATABASE.getCode()) + .build(); public static final Set ORACLE_SYSTEM_DATABASES = - Sets.newHashSet("SYS", "SYSTEM", "SYSDBA", "SYSOPER", "HR", "SCOTT"); + Sets.newHashSet("SYS", "SYSTEM", "SYSDBA", "SYSOPER", "HR", "SCOTT"); public static final OptionRule OPTION_RULE = - OptionRule.builder() - .required(OracleOptionRule.URL, OracleOptionRule.DRIVER) - .optional(OracleOptionRule.USER, OracleOptionRule.PASSWORD) - .build(); + OptionRule.builder() + .required(OracleOptionRule.URL, OracleOptionRule.DRIVER) + .optional(OracleOptionRule.USER, OracleOptionRule.PASSWORD) + .build(); public static final OptionRule METADATA_RULE = - OptionRule.builder() - .required(OracleOptionRule.DATABASE, OracleOptionRule.TABLE) - .build(); + OptionRule.builder() + .required(OracleOptionRule.DATABASE, OracleOptionRule.TABLE) + .build(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleOptionRule.java index 517a8c32e..f3ec40e33 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleOptionRule.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleOptionRule.java @@ -23,28 +23,28 @@ public class OracleOptionRule { public static final Option URL = - Options.key("url") - .stringType() - .noDefaultValue() - .withDescription("jdbc url, eg:" + "jdbc:oracle:thin:@localhost:1521:XE"); + Options.key("url") + .stringType() + .noDefaultValue() + .withDescription("jdbc url, eg:" + "jdbc:oracle:thin:@localhost:1521:XE"); public static final Option USER = - Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); + Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); public static final Option PASSWORD = - Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); + Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); public static final Option DATABASE = - Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); + Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); public static final Option TABLE = - Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); + Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); public static final Option DRIVER = - Options.key("driver") - .enumType(DriverType.class) - .defaultValue(DriverType.ORACLE) - .withDescription("driver"); + Options.key("driver") + .enumType(DriverType.class) + .defaultValue(DriverType.ORACLE) + .withDescription("driver"); public enum DriverType { ORACLE("oracle.jdbc.driver.OracleDriver"), diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java index b331d61d0..e43939536 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java @@ -17,17 +17,16 @@ package org.apache.seatunnel.datasource.plugin.postgresql.jdbc; -import static com.google.common.base.Preconditions.checkNotNull; - import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; -import lombok.NonNull; import org.apache.commons.lang3.StringUtils; +import lombok.NonNull; + import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.DriverManager; @@ -39,6 +38,8 @@ import java.util.List; import java.util.Map; +import static com.google.common.base.Preconditions.checkNotNull; + public class PostgresqlDataSourceChannel implements DataSourceChannel { @Override @@ -53,18 +54,18 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { List tableNames = new ArrayList<>(); String query = "SELECT table_schema, table_name FROM information_schema.tables"; try (Connection connection = getConnection(requestParams, database)) { try (Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(query)) { + ResultSet resultSet = statement.executeQuery(query)) { while (resultSet.next()) { String schemaName = resultSet.getString("table_schema"); String tableName = resultSet.getString("table_name"); if (StringUtils.isNotBlank(schemaName) - && !PostgresqlDataSourceConfig.POSTGRESQL_SYSTEM_DATABASES.contains( - schemaName)) { + && !PostgresqlDataSourceConfig.POSTGRESQL_SYSTEM_DATABASES.contains( + schemaName)) { tableNames.add(schemaName + "." + tableName); } } @@ -77,17 +78,17 @@ public List getTables( @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { List dbNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - PreparedStatement statement = - connection.prepareStatement("select datname from pg_database;"); - ResultSet re = statement.executeQuery()) { + PreparedStatement statement = + connection.prepareStatement("select datname from pg_database;"); + ResultSet re = statement.executeQuery()) { while (re.next()) { String dbName = re.getString("datname"); if (StringUtils.isNotBlank(dbName) - && !PostgresqlDataSourceConfig.POSTGRESQL_SYSTEM_DATABASES.contains( - dbName)) { + && !PostgresqlDataSourceConfig.POSTGRESQL_SYSTEM_DATABASES.contains( + dbName)) { dbNames.add(dbName); } } @@ -99,7 +100,7 @@ public List getDatabases( @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try (Connection ignored = getConnection(requestParams)) { return true; } catch (Exception e) { @@ -109,18 +110,18 @@ public boolean checkDataSourceConnectivity( @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { List tableFields = new ArrayList<>(); - try (Connection connection = getConnection(requestParams, database);) { + try (Connection connection = getConnection(requestParams, database); ) { DatabaseMetaData metaData = connection.getMetaData(); String primaryKey = getPrimaryKey(metaData, database, table); String[] split = table.split("\\."); if (split.length != 2) { throw new DataSourcePluginException( - "Postgresql tableName should composed by schemaName.tableName"); + "Postgresql tableName should composed by schemaName.tableName"); } try (ResultSet resultSet = metaData.getColumns(database, split[0], split[1], null)) { while (resultSet.next()) { @@ -146,15 +147,15 @@ public List getTableFields( @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { return null; } private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { + throws SQLException { ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); while (primaryKeysInfo.next()) { return primaryKeysInfo.getString("COLUMN_NAME"); @@ -163,17 +164,17 @@ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String ta } private Connection getConnection(Map requestParams) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { return getConnection(requestParams, null); } private Connection getConnection(Map requestParams, String databaseName) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { checkNotNull(requestParams.get(PostgresqlOptionRule.DRIVER.key())); checkNotNull(requestParams.get(PostgresqlOptionRule.URL.key()), "Jdbc url cannot be null"); String url = - JdbcUtils.replaceDatabase( - requestParams.get(PostgresqlOptionRule.URL.key()), databaseName); + JdbcUtils.replaceDatabase( + requestParams.get(PostgresqlOptionRule.URL.key()), databaseName); if (requestParams.containsKey(PostgresqlOptionRule.USER.key())) { String username = requestParams.get(PostgresqlOptionRule.USER.key()); String password = requestParams.get(PostgresqlOptionRule.PASSWORD.key()); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceConfig.java index 2aa0538b2..f150ab5d0 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceConfig.java @@ -30,33 +30,33 @@ public class PostgresqlDataSourceConfig { public static final String PLUGIN_NAME = "JDBC-Postgres"; public static final DataSourcePluginInfo POSTGRESQL_DATASOURCE_PLUGIN_INFO = - DataSourcePluginInfo.builder() - .name(PLUGIN_NAME) - .icon(PLUGIN_NAME) - .version("1.0.0") - .type(DatasourcePluginTypeEnum.DATABASE.getCode()) - .build(); + DataSourcePluginInfo.builder() + .name(PLUGIN_NAME) + .icon(PLUGIN_NAME) + .version("1.0.0") + .type(DatasourcePluginTypeEnum.DATABASE.getCode()) + .build(); public static final Set POSTGRESQL_SYSTEM_DATABASES = - Sets.newHashSet( - "information_schema", - "pg_catalog", - "root", - "pg_toast", - "pg_temp_1", - "pg_toast_temp_1", - "postgres", - "template0", - "template1"); + Sets.newHashSet( + "information_schema", + "pg_catalog", + "root", + "pg_toast", + "pg_temp_1", + "pg_toast_temp_1", + "postgres", + "template0", + "template1"); public static final OptionRule OPTION_RULE = - OptionRule.builder() - .required(PostgresqlOptionRule.URL, PostgresqlOptionRule.DRIVER) - .optional(PostgresqlOptionRule.USER, PostgresqlOptionRule.PASSWORD) - .build(); + OptionRule.builder() + .required(PostgresqlOptionRule.URL, PostgresqlOptionRule.DRIVER) + .optional(PostgresqlOptionRule.USER, PostgresqlOptionRule.PASSWORD) + .build(); public static final OptionRule METADATA_RULE = - OptionRule.builder() - .required(PostgresqlOptionRule.DATABASE, PostgresqlOptionRule.TABLE) - .build(); + OptionRule.builder() + .required(PostgresqlOptionRule.DATABASE, PostgresqlOptionRule.TABLE) + .build(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlOptionRule.java index 16e8b85b2..748c4ea79 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlOptionRule.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlOptionRule.java @@ -23,30 +23,30 @@ public class PostgresqlOptionRule { public static final Option URL = - Options.key("url") - .stringType() - .noDefaultValue() - .withDescription( - "jdbc url, eg:" - + "jdbc:postgresql://localhost:5432//test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); + Options.key("url") + .stringType() + .noDefaultValue() + .withDescription( + "jdbc url, eg:" + + "jdbc:postgresql://localhost:5432//test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); public static final Option USER = - Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); + Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); public static final Option PASSWORD = - Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); + Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); public static final Option DATABASE = - Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); + Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); public static final Option TABLE = - Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); + Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); public static final Option DRIVER = - Options.key("driver") - .enumType(DriverType.class) - .defaultValue(DriverType.POSTGRESQL) - .withDescription("driver"); + Options.key("driver") + .enumType(DriverType.class) + .defaultValue(DriverType.POSTGRESQL) + .withDescription("driver"); public enum DriverType { POSTGRESQL("org.postgresql.Driver"), diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceChannel.java index 32f77e985..0e3dfa709 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceChannel.java @@ -17,17 +17,16 @@ package org.apache.seatunnel.datasource.plugin.redshift.jdbc; -import static com.google.common.base.Preconditions.checkNotNull; - import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; -import lombok.NonNull; import org.apache.commons.lang3.StringUtils; +import lombok.NonNull; + import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.DriverManager; @@ -38,6 +37,8 @@ import java.util.List; import java.util.Map; +import static com.google.common.base.Preconditions.checkNotNull; + public class RedshiftDataSourceChannel implements DataSourceChannel { @Override @@ -52,17 +53,17 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { List tableNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams, database); - ResultSet resultSet = - connection.getMetaData().getTables(database, null, null, null);) { + ResultSet resultSet = + connection.getMetaData().getTables(database, null, null, null); ) { while (resultSet.next()) { String schemaName = resultSet.getString("TABLE_SCHEM"); String tableName = resultSet.getString("TABLE_NAME"); // todo: use isNotSystemSchemaName if (StringUtils.isNotBlank(schemaName) - && !RedshiftDataSourceConfig.REDSHIFT_SYSTEM_TABLES.contains(schemaName)) { + && !RedshiftDataSourceConfig.REDSHIFT_SYSTEM_TABLES.contains(schemaName)) { tableNames.add(schemaName + "." + tableName); } } @@ -74,16 +75,16 @@ public List getTables( @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { List dbNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - PreparedStatement statement = - connection.prepareStatement("select datname from pg_database;"); - ResultSet re = statement.executeQuery()) { + PreparedStatement statement = + connection.prepareStatement("select datname from pg_database;"); + ResultSet re = statement.executeQuery()) { while (re.next()) { String dbName = re.getString("datname"); if (StringUtils.isNotBlank(dbName) - && !RedshiftDataSourceConfig.REDSHIFT_SYSTEM_TABLES.contains(dbName)) { + && !RedshiftDataSourceConfig.REDSHIFT_SYSTEM_TABLES.contains(dbName)) { dbNames.add(dbName); } } @@ -95,7 +96,7 @@ public List getDatabases( @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try (Connection ignored = getConnection(requestParams)) { return true; } catch (Exception e) { @@ -105,18 +106,18 @@ public boolean checkDataSourceConnectivity( @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { List tableFields = new ArrayList<>(); - try (Connection connection = getConnection(requestParams, database);) { + try (Connection connection = getConnection(requestParams, database); ) { DatabaseMetaData metaData = connection.getMetaData(); String primaryKey = getPrimaryKey(metaData, database, table); String[] split = table.split("\\."); if (split.length != 2) { throw new DataSourcePluginException( - "Postgresql tableName should composed by schemaName.tableName"); + "Postgresql tableName should composed by schemaName.tableName"); } try (ResultSet resultSet = metaData.getColumns(database, split[0], split[1], null)) { while (resultSet.next()) { @@ -142,15 +143,15 @@ public List getTableFields( @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { return null; } private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { + throws SQLException { ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); while (primaryKeysInfo.next()) { return primaryKeysInfo.getString("COLUMN_NAME"); @@ -159,17 +160,17 @@ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String ta } private Connection getConnection(Map requestParams) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { return getConnection(requestParams, null); } private Connection getConnection(Map requestParams, String databaseName) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { checkNotNull(requestParams.get(RedshiftOptionRule.DRIVER.key())); checkNotNull(requestParams.get(RedshiftOptionRule.URL.key()), "Jdbc url cannot be null"); String url = - JdbcUtils.replaceDatabase( - requestParams.get(RedshiftOptionRule.URL.key()), databaseName); + JdbcUtils.replaceDatabase( + requestParams.get(RedshiftOptionRule.URL.key()), databaseName); if (requestParams.containsKey(RedshiftOptionRule.USER.key())) { String username = requestParams.get(RedshiftOptionRule.USER.key()); String password = requestParams.get(RedshiftOptionRule.PASSWORD.key()); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceConfig.java index 450c8d954..b5561f09e 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceConfig.java @@ -30,33 +30,33 @@ public class RedshiftDataSourceConfig { public static final String PLUGIN_NAME = "JDBC-Redshift"; public static final DataSourcePluginInfo REDSHIFT_DATASOURCE_PLUGIN_INFO = - DataSourcePluginInfo.builder() - .name(PLUGIN_NAME) - .icon("redshift") - .version("1.0.0") - .type(DatasourcePluginTypeEnum.DATABASE.getCode()) - .build(); + DataSourcePluginInfo.builder() + .name(PLUGIN_NAME) + .icon("redshift") + .version("1.0.0") + .type(DatasourcePluginTypeEnum.DATABASE.getCode()) + .build(); public static final Set REDSHIFT_SYSTEM_TABLES = - Sets.newHashSet( - "information_schema", - "pg_catalog", - "root", - "pg_toast", - "pg_temp_1", - "pg_toast_temp_1", - "postgres", - "template0", - "template1"); + Sets.newHashSet( + "information_schema", + "pg_catalog", + "root", + "pg_toast", + "pg_temp_1", + "pg_toast_temp_1", + "postgres", + "template0", + "template1"); public static final OptionRule OPTION_RULE = - OptionRule.builder() - .required(RedshiftOptionRule.URL, RedshiftOptionRule.DRIVER) - .optional(RedshiftOptionRule.USER, RedshiftOptionRule.PASSWORD) - .build(); + OptionRule.builder() + .required(RedshiftOptionRule.URL, RedshiftOptionRule.DRIVER) + .optional(RedshiftOptionRule.USER, RedshiftOptionRule.PASSWORD) + .build(); public static final OptionRule METADATA_RULE = - OptionRule.builder() - .required(RedshiftOptionRule.DATABASE, RedshiftOptionRule.TABLE) - .build(); + OptionRule.builder() + .required(RedshiftOptionRule.DATABASE, RedshiftOptionRule.TABLE) + .build(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftOptionRule.java index ba4c11c0f..c4f389545 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftOptionRule.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftOptionRule.java @@ -23,30 +23,30 @@ public class RedshiftOptionRule { public static final Option URL = - Options.key("url") - .stringType() - .noDefaultValue() - .withDescription( - "jdbc url, eg:" - + "jdbc:redshift://server.redshift.amazonaws.com:5439/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); + Options.key("url") + .stringType() + .noDefaultValue() + .withDescription( + "jdbc url, eg:" + + "jdbc:redshift://server.redshift.amazonaws.com:5439/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); public static final Option USER = - Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); + Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); public static final Option PASSWORD = - Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); + Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); public static final Option DATABASE = - Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); + Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); public static final Option TABLE = - Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); + Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); public static final Option DRIVER = - Options.key("driver") - .enumType(DriverType.class) - .defaultValue(DriverType.JDBC42_REDSHIFT) - .withDescription("driver"); + Options.key("driver") + .enumType(DriverType.class) + .defaultValue(DriverType.JDBC42_REDSHIFT) + .withDescription("driver"); public enum DriverType { JDBC42_REDSHIFT("com.amazon.redshift.jdbc42.Driver"), diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java index bde7550c9..7a32f86a0 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java @@ -17,17 +17,16 @@ package org.apache.seatunnel.datasource.plugin.sqlserver.jdbc; -import static com.google.common.base.Preconditions.checkNotNull; - import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; +import org.apache.commons.lang3.StringUtils; + import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -39,6 +38,8 @@ import java.util.List; import java.util.Map; +import static com.google.common.base.Preconditions.checkNotNull; + @Slf4j public class SqlServerDataSourceChannel implements DataSourceChannel { @Override @@ -53,13 +54,13 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { List tableNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - ResultSet resultSet = - connection - .getMetaData() - .getTables(database, null, null, new String[]{"TABLE"})) { + ResultSet resultSet = + connection + .getMetaData() + .getTables(database, null, null, new String[] {"TABLE"})) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); if (StringUtils.isNotBlank(tableName)) { @@ -74,16 +75,16 @@ public List getTables( @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { List dbNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); - ResultSet re = statement.executeQuery()) { + PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); + ResultSet re = statement.executeQuery()) { // filter system databases while (re.next()) { String dbName = re.getString("database"); if (StringUtils.isNotBlank(dbName) - && !SqlServerDataSourceConfig.SQLSERVER_SYSTEM_DATABASES.contains(dbName)) { + && !SqlServerDataSourceConfig.SQLSERVER_SYSTEM_DATABASES.contains(dbName)) { dbNames.add(dbName); } } @@ -95,7 +96,7 @@ public List getDatabases( @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try (Connection ignored = getConnection(requestParams)) { return true; } catch (Exception e) { @@ -105,10 +106,10 @@ public boolean checkDataSourceConnectivity( @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { List tableFields = new ArrayList<>(); try (Connection connection = getConnection(requestParams, database)) { DatabaseMetaData metaData = connection.getMetaData(); @@ -137,15 +138,15 @@ public List getTableFields( @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { return null; } private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { + throws SQLException { ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); while (primaryKeysInfo.next()) { return primaryKeysInfo.getString("COLUMN_NAME"); @@ -154,17 +155,17 @@ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String ta } private Connection getConnection(Map requestParams) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { return getConnection(requestParams, null); } private Connection getConnection(Map requestParams, String databaseName) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { checkNotNull(requestParams.get(SqlServerOptionRule.DRIVER.key())); checkNotNull(requestParams.get(SqlServerOptionRule.URL.key()), "Jdbc url cannot be null"); String url = - JdbcUtils.replaceDatabase( - requestParams.get(SqlServerOptionRule.URL.key()), databaseName); + JdbcUtils.replaceDatabase( + requestParams.get(SqlServerOptionRule.URL.key()), databaseName); if (requestParams.containsKey(SqlServerOptionRule.USER.key())) { String username = requestParams.get(SqlServerOptionRule.USER.key()); String password = requestParams.get(SqlServerOptionRule.PASSWORD.key()); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceConfig.java index 3bfc056c1..218464abe 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceConfig.java @@ -30,31 +30,31 @@ public class SqlServerDataSourceConfig { public static final String PLUGIN_NAME = "JDBC-SQLServer"; public static final DataSourcePluginInfo SQLSERVER_DATASOURCE_PLUGIN_INFO = - DataSourcePluginInfo.builder() - .name(PLUGIN_NAME) - .icon(PLUGIN_NAME) - .version("1.0.0") - .type(DatasourcePluginTypeEnum.DATABASE.getCode()) - .build(); + DataSourcePluginInfo.builder() + .name(PLUGIN_NAME) + .icon(PLUGIN_NAME) + .version("1.0.0") + .type(DatasourcePluginTypeEnum.DATABASE.getCode()) + .build(); public static final Set SQLSERVER_SYSTEM_DATABASES = - Sets.newHashSet( - "master", - "tempdb", - "model", - "msdb", - "ReportServer", - "ReportServerTempDB", - "SSISDB"); + Sets.newHashSet( + "master", + "tempdb", + "model", + "msdb", + "ReportServer", + "ReportServerTempDB", + "SSISDB"); public static final OptionRule OPTION_RULE = - OptionRule.builder() - .required(SqlServerOptionRule.URL, SqlServerOptionRule.DRIVER) - .optional(SqlServerOptionRule.USER, SqlServerOptionRule.PASSWORD) - .build(); + OptionRule.builder() + .required(SqlServerOptionRule.URL, SqlServerOptionRule.DRIVER) + .optional(SqlServerOptionRule.USER, SqlServerOptionRule.PASSWORD) + .build(); public static final OptionRule METADATA_RULE = - OptionRule.builder() - .required(SqlServerOptionRule.DATABASE, SqlServerOptionRule.TABLE) - .build(); + OptionRule.builder() + .required(SqlServerOptionRule.DATABASE, SqlServerOptionRule.TABLE) + .build(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerOptionRule.java index ec157d39c..6052a7ab1 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerOptionRule.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerOptionRule.java @@ -23,29 +23,29 @@ public class SqlServerOptionRule { public static final Option URL = - Options.key("url") - .stringType() - .noDefaultValue() - .withDescription( - "jdbc url, eg:" + "jdbc:sqlserver://localhost:1433;database=xx"); + Options.key("url") + .stringType() + .noDefaultValue() + .withDescription( + "jdbc url, eg:" + "jdbc:sqlserver://localhost:1433;database=xx"); public static final Option USER = - Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); + Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); public static final Option PASSWORD = - Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); + Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); public static final Option DATABASE = - Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); + Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); public static final Option TABLE = - Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); + Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); public static final Option DRIVER = - Options.key("driver") - .enumType(DriverType.class) - .defaultValue(DriverType.SQL_SERVER) - .withDescription("driver"); + Options.key("driver") + .enumType(DriverType.class) + .defaultValue(DriverType.SQL_SERVER) + .withDescription("driver"); public enum DriverType { SQL_SERVER("com.microsoft.sqlserver.jdbc.SQLServerDriver"), diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksDataSourceConfig.java index 427657ccc..913bc5ced 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksDataSourceConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksDataSourceConfig.java @@ -30,24 +30,24 @@ public class StarRocksDataSourceConfig { public static final String PLUGIN_NAME = "JDBC-StarRocks"; public static final DataSourcePluginInfo STAR_ROCKS_DATA_SOURCE_PLUGIN_INFO = - DataSourcePluginInfo.builder() - .name(PLUGIN_NAME) - .icon(PLUGIN_NAME) - .version("1.0.0") - .type(DatasourcePluginTypeEnum.DATABASE.getCode()) - .build(); + DataSourcePluginInfo.builder() + .name(PLUGIN_NAME) + .icon(PLUGIN_NAME) + .version("1.0.0") + .type(DatasourcePluginTypeEnum.DATABASE.getCode()) + .build(); public static final Set STAR_ROCKS_SYSTEM_DATABASES = - Sets.newHashSet("_statistics_", "information_schema"); + Sets.newHashSet("_statistics_", "information_schema"); public static final OptionRule OPTION_RULE = - OptionRule.builder() - .required(StarRocksOptionRule.URL, StarRocksOptionRule.DRIVER) - .optional(StarRocksOptionRule.USER, StarRocksOptionRule.PASSWORD) - .build(); + OptionRule.builder() + .required(StarRocksOptionRule.URL, StarRocksOptionRule.DRIVER) + .optional(StarRocksOptionRule.USER, StarRocksOptionRule.PASSWORD) + .build(); public static final OptionRule METADATA_RULE = - OptionRule.builder() - .required(StarRocksOptionRule.DATABASE, StarRocksOptionRule.TABLE) - .build(); + OptionRule.builder() + .required(StarRocksOptionRule.DATABASE, StarRocksOptionRule.TABLE) + .build(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java index 3f429a7cc..65f0f6136 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java @@ -17,17 +17,16 @@ package org.apache.seatunnel.datasource.plugin.starrocks.jdbc; -import static com.google.common.base.Preconditions.checkNotNull; - import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; -import lombok.NonNull; import org.apache.commons.lang3.StringUtils; +import lombok.NonNull; + import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.DriverManager; @@ -38,6 +37,8 @@ import java.util.List; import java.util.Map; +import static com.google.common.base.Preconditions.checkNotNull; + public class StarRocksJdbcDataSourceChannel implements DataSourceChannel { @Override @@ -52,13 +53,13 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { List tableNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - ResultSet resultSet = - connection - .getMetaData() - .getTables(database, null, null, new String[]{"TABLE"})) { + ResultSet resultSet = + connection + .getMetaData() + .getTables(database, null, null, new String[] {"TABLE"})) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); if (StringUtils.isNotBlank(tableName)) { @@ -73,17 +74,17 @@ public List getTables( @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { List dbNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); - ResultSet re = statement.executeQuery()) { + PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); + ResultSet re = statement.executeQuery()) { // filter system databases while (re.next()) { String dbName = re.getString("database"); if (StringUtils.isNotBlank(dbName) - && !StarRocksDataSourceConfig.STAR_ROCKS_SYSTEM_DATABASES.contains( - dbName)) { + && !StarRocksDataSourceConfig.STAR_ROCKS_SYSTEM_DATABASES.contains( + dbName)) { dbNames.add(dbName); } } @@ -95,7 +96,7 @@ public List getDatabases( @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try (Connection ignored = getConnection(requestParams)) { return true; } catch (Exception e) { @@ -105,15 +106,15 @@ public boolean checkDataSourceConnectivity( @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { List tableFields = new ArrayList<>(); try (Connection connection = getConnection(requestParams, database)) { DatabaseMetaData metaData = connection.getMetaData(); String primaryKey = getPrimaryKey(metaData, database, table); - try (ResultSet resultSet = metaData.getColumns(database, null, table, null);) { + try (ResultSet resultSet = metaData.getColumns(database, null, table, null); ) { while (resultSet.next()) { TableField tableField = new TableField(); String columnName = resultSet.getString("COLUMN_NAME"); @@ -137,15 +138,15 @@ public List getTableFields( @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { return null; } private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { + throws SQLException { ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); while (primaryKeysInfo.next()) { return primaryKeysInfo.getString("COLUMN_NAME"); @@ -154,17 +155,17 @@ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String ta } private Connection getConnection(Map requestParams) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { return getConnection(requestParams, null); } private Connection getConnection(Map requestParams, String databaseName) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { checkNotNull(requestParams.get(StarRocksOptionRule.DRIVER.key())); checkNotNull(requestParams.get(StarRocksOptionRule.URL.key()), "Jdbc url cannot be null"); String url = - JdbcUtils.replaceDatabase( - requestParams.get(StarRocksOptionRule.URL.key()), databaseName); + JdbcUtils.replaceDatabase( + requestParams.get(StarRocksOptionRule.URL.key()), databaseName); if (requestParams.containsKey(StarRocksOptionRule.USER.key())) { String username = requestParams.get(StarRocksOptionRule.USER.key()); String password = requestParams.get(StarRocksOptionRule.PASSWORD.key()); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksOptionRule.java index d67c2a304..08f2dc21b 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksOptionRule.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksOptionRule.java @@ -23,30 +23,30 @@ public class StarRocksOptionRule { public static final Option URL = - Options.key("url") - .stringType() - .noDefaultValue() - .withDescription( - "jdbc url, eg:" - + "jdbc:mysql://localhost:9030/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); + Options.key("url") + .stringType() + .noDefaultValue() + .withDescription( + "jdbc url, eg:" + + "jdbc:mysql://localhost:9030/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); public static final Option USER = - Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); + Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); public static final Option PASSWORD = - Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); + Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); public static final Option DATABASE = - Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); + Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); public static final Option TABLE = - Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); + Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); public static final Option DRIVER = - Options.key("driver") - .enumType(DriverType.class) - .defaultValue(DriverType.MYSQL) - .withDescription("driver"); + Options.key("driver") + .enumType(DriverType.class) + .defaultValue(DriverType.MYSQL) + .withDescription("driver"); public enum DriverType { MYSQL("com.mysql.cj.jdbc.Driver"), diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbDataSourceConfig.java index f02caff2f..c9acb5b6b 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbDataSourceConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbDataSourceConfig.java @@ -30,22 +30,22 @@ public class TidbDataSourceConfig { public static final String PLUGIN_NAME = "JDBC-TiDB"; public static final DataSourcePluginInfo TIDB_DATASOURCE_PLUGIN_INFO = - DataSourcePluginInfo.builder() - .name(PLUGIN_NAME) - .icon(PLUGIN_NAME) - .version("1.0.0") - .type(DatasourcePluginTypeEnum.DATABASE.getCode()) - .build(); + DataSourcePluginInfo.builder() + .name(PLUGIN_NAME) + .icon(PLUGIN_NAME) + .version("1.0.0") + .type(DatasourcePluginTypeEnum.DATABASE.getCode()) + .build(); public static final Set TIDB_SYSTEM_DATABASES = - Sets.newHashSet("information_schema", "mysql", "performance_schema", "metrics_schema"); + Sets.newHashSet("information_schema", "mysql", "performance_schema", "metrics_schema"); public static final OptionRule OPTION_RULE = - OptionRule.builder() - .required(TidbOptionRule.URL, TidbOptionRule.DRIVER) - .optional(TidbOptionRule.USER, TidbOptionRule.PASSWORD) - .build(); + OptionRule.builder() + .required(TidbOptionRule.URL, TidbOptionRule.DRIVER) + .optional(TidbOptionRule.USER, TidbOptionRule.PASSWORD) + .build(); public static final OptionRule METADATA_RULE = - OptionRule.builder().required(TidbOptionRule.DATABASE, TidbOptionRule.TABLE).build(); + OptionRule.builder().required(TidbOptionRule.DATABASE, TidbOptionRule.TABLE).build(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java index 8558f13c0..90688bbb5 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java @@ -17,17 +17,16 @@ package org.apache.seatunnel.datasource.plugin.tidb.jdbc; -import static com.google.common.base.Preconditions.checkNotNull; - import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; -import lombok.NonNull; import org.apache.commons.lang3.StringUtils; +import lombok.NonNull; + import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.DriverManager; @@ -40,6 +39,8 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static com.google.common.base.Preconditions.checkNotNull; + public class TidbJdbcDataSourceChannel implements DataSourceChannel { @Override @@ -54,13 +55,13 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { List tableNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - ResultSet resultSet = - connection - .getMetaData() - .getTables(database, null, null, new String[]{"TABLE"})) { + ResultSet resultSet = + connection + .getMetaData() + .getTables(database, null, null, new String[] {"TABLE"})) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); if (StringUtils.isNotBlank(tableName)) { @@ -75,16 +76,16 @@ public List getTables( @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { List dbNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); - ResultSet re = statement.executeQuery()) { + PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); + ResultSet re = statement.executeQuery()) { // filter system databases while (re.next()) { String dbName = re.getString("database"); if (StringUtils.isNotBlank(dbName) - && !TidbDataSourceConfig.TIDB_SYSTEM_DATABASES.contains(dbName)) { + && !TidbDataSourceConfig.TIDB_SYSTEM_DATABASES.contains(dbName)) { dbNames.add(dbName); } } @@ -96,7 +97,7 @@ public List getDatabases( @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try (Connection ignored = getConnection(requestParams)) { return true; } catch (Exception e) { @@ -106,10 +107,10 @@ public boolean checkDataSourceConnectivity( @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { List tableFields = new ArrayList<>(); try (Connection connection = getConnection(requestParams, database)) { DatabaseMetaData metaData = connection.getMetaData(); @@ -138,21 +139,21 @@ public List getTableFields( @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { return tables.parallelStream() - .collect( - Collectors.toMap( - Function.identity(), - table -> - getTableFields( - pluginName, requestParams, database, table))); + .collect( + Collectors.toMap( + Function.identity(), + table -> + getTableFields( + pluginName, requestParams, database, table))); } private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { + throws SQLException { ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); while (primaryKeysInfo.next()) { return primaryKeysInfo.getString("COLUMN_NAME"); @@ -161,17 +162,17 @@ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String ta } private Connection getConnection(Map requestParams) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { return getConnection(requestParams, null); } private Connection getConnection(Map requestParams, String databaseName) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { checkNotNull(requestParams.get(TidbOptionRule.DRIVER.key())); checkNotNull(requestParams.get(TidbOptionRule.URL.key()), "Jdbc url cannot be null"); String url = - JdbcUtils.replaceDatabase( - requestParams.get(TidbOptionRule.URL.key()), databaseName); + JdbcUtils.replaceDatabase( + requestParams.get(TidbOptionRule.URL.key()), databaseName); if (requestParams.containsKey(TidbOptionRule.USER.key())) { String username = requestParams.get(TidbOptionRule.USER.key()); String password = requestParams.get(TidbOptionRule.PASSWORD.key()); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbOptionRule.java index 90af0f95f..7ae009170 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbOptionRule.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbOptionRule.java @@ -23,30 +23,30 @@ public class TidbOptionRule { public static final Option URL = - Options.key("url") - .stringType() - .noDefaultValue() - .withDescription( - "jdbc url, eg:" - + " jdbc:mysql://localhost:3306/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); + Options.key("url") + .stringType() + .noDefaultValue() + .withDescription( + "jdbc url, eg:" + + " jdbc:mysql://localhost:3306/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); public static final Option USER = - Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); + Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); public static final Option PASSWORD = - Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); + Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); public static final Option DATABASE = - Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); + Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); public static final Option TABLE = - Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); + Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); public static final Option DRIVER = - Options.key("driver") - .enumType(DriverType.class) - .defaultValue(DriverType.MYSQL) - .withDescription("driver"); + Options.key("driver") + .enumType(DriverType.class) + .defaultValue(DriverType.MYSQL) + .withDescription("driver"); public enum DriverType { MYSQL("com.mysql.cj.jdbc.Driver"), diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannel.java index 72e679957..9413bcae1 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannel.java @@ -17,33 +17,34 @@ package org.apache.seatunnel.datasource.plugin.kafka; -import static com.google.common.base.Preconditions.checkArgument; - import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.DescribeClusterOptions; import org.apache.kafka.clients.admin.DescribeClusterResult; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; +import static com.google.common.base.Preconditions.checkArgument; + @Slf4j public class KafkaDataSourceChannel implements DataSourceChannel { private static final String DATABASE = "default"; private static final DescribeClusterOptions DEFAULT_TIMEOUT_OPTIONS = - new DescribeClusterOptions().timeoutMs(60 * 1000); + new DescribeClusterOptions().timeoutMs(60 * 1000); @Override public OptionRule getDataSourceOptions(@NonNull String pluginName) { @@ -57,59 +58,59 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); try (AdminClient adminClient = createAdminClient(requestParams)) { Set strings = adminClient.listTopics().names().get(); return new ArrayList<>(strings); } catch (Exception ex) { throw new DataSourcePluginException( - "check kafka connectivity failed, " + ex.getMessage(), ex); + "check kafka connectivity failed, " + ex.getMessage(), ex); } } @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { return DEFAULT_DATABASES; } @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try (AdminClient adminClient = createAdminClient(requestParams)) { // just test the connection DescribeClusterResult describeClusterResult = - adminClient.describeCluster(DEFAULT_TIMEOUT_OPTIONS); + adminClient.describeCluster(DEFAULT_TIMEOUT_OPTIONS); return CollectionUtils.isNotEmpty(describeClusterResult.nodes().get()); } catch (Exception ex) { throw new DataSourcePluginException( - "check kafka connectivity failed, " + ex.getMessage(), ex); + "check kafka connectivity failed, " + ex.getMessage(), ex); } } @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); return Collections.emptyList(); } @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); return Collections.emptyMap(); } private AdminClient createAdminClient(Map requestParams) { return AdminClient.create( - KafkaRequestParamsUtils.parsePropertiesFromRequestParams(requestParams)); + KafkaRequestParamsUtils.parsePropertiesFromRequestParams(requestParams)); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceFactory.java index bcba81c0e..16b258cdf 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceFactory.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceFactory.java @@ -42,13 +42,13 @@ public String factoryIdentifier() { @Override public Set supportedDataSources() { return Sets.newHashSet( - DataSourcePluginInfo.builder() - .name(KAFKA_PLUGIN_NAME) - .icon(KAFKA_PLUGIN_ICON) - .version(KAFKA_PLUGIN_VERSION) - .supportVirtualTables(true) - .type(DatasourcePluginTypeEnum.NO_STRUCTURED.getCode()) - .build()); + DataSourcePluginInfo.builder() + .name(KAFKA_PLUGIN_NAME) + .icon(KAFKA_PLUGIN_ICON) + .version(KAFKA_PLUGIN_VERSION) + .supportVirtualTables(true) + .type(DatasourcePluginTypeEnum.NO_STRUCTURED.getCode()) + .build()); } @Override diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtils.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtils.java index 95988e762..25e70c991 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtils.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtils.java @@ -17,8 +17,6 @@ package org.apache.seatunnel.datasource.plugin.kafka; -import static com.google.common.base.Preconditions.checkArgument; - import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; @@ -27,28 +25,30 @@ import java.util.Map; import java.util.Properties; +import static com.google.common.base.Preconditions.checkArgument; + public class KafkaRequestParamsUtils { public static Properties parsePropertiesFromRequestParams(Map requestParams) { checkArgument( - requestParams.containsKey(KafkaOptionRule.BOOTSTRAP_SERVERS.key()), - String.format( - "Missing %s in requestParams", KafkaOptionRule.BOOTSTRAP_SERVERS.key())); + requestParams.containsKey(KafkaOptionRule.BOOTSTRAP_SERVERS.key()), + String.format( + "Missing %s in requestParams", KafkaOptionRule.BOOTSTRAP_SERVERS.key())); final Properties properties = new Properties(); properties.put( - AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, - requestParams.get(KafkaOptionRule.BOOTSTRAP_SERVERS.key())); + AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, + requestParams.get(KafkaOptionRule.BOOTSTRAP_SERVERS.key())); if (requestParams.containsKey(KafkaOptionRule.KAFKA_CONFIG.key())) { Config configObject = - ConfigFactory.parseString( - requestParams.get(KafkaOptionRule.KAFKA_CONFIG.key())); + ConfigFactory.parseString( + requestParams.get(KafkaOptionRule.KAFKA_CONFIG.key())); configObject - .entrySet() - .forEach( - entry -> { - properties.put( - entry.getKey(), entry.getValue().unwrapped().toString()); - }); + .entrySet() + .forEach( + entry -> { + properties.put( + entry.getKey(), entry.getValue().unwrapped().toString()); + }); } return properties; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannelTest.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannelTest.java index e694de45a..cdcb4ffa9 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannelTest.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannelTest.java @@ -20,12 +20,13 @@ import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.plugin.api.model.TableField; -import com.google.common.collect.ImmutableMap; -import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import com.google.common.collect.ImmutableMap; +import lombok.extern.slf4j.Slf4j; + import java.util.Collections; import java.util.List; import java.util.Map; @@ -36,37 +37,37 @@ public class KafkaDataSourceChannelTest { private static final KafkaDataSourceChannel KAFKA_DATA_SOURCE_CHANNEL = - new KafkaDataSourceChannel(); + new KafkaDataSourceChannel(); private static final String KAFKA_PLUGIN_NAME = "kafka"; private static final String BOOTSTRAP_SERVER = "localhost:9092"; private static final Map REQUEST_PARAMS = - new ImmutableMap.Builder() - .put(KafkaOptionRule.BOOTSTRAP_SERVERS.key(), BOOTSTRAP_SERVER) - .build(); + new ImmutableMap.Builder() + .put(KafkaOptionRule.BOOTSTRAP_SERVERS.key(), BOOTSTRAP_SERVER) + .build(); @Test public void getDataSourceOptions() { OptionRule dataSourceMetadataFieldsByDataSourceName = - KAFKA_DATA_SOURCE_CHANNEL.getDataSourceOptions(KAFKA_PLUGIN_NAME); + KAFKA_DATA_SOURCE_CHANNEL.getDataSourceOptions(KAFKA_PLUGIN_NAME); Assertions.assertEquals( - 1, dataSourceMetadataFieldsByDataSourceName.getRequiredOptions().size()); + 1, dataSourceMetadataFieldsByDataSourceName.getRequiredOptions().size()); } @Test public void getDatasourceMetadataFieldsByDataSourceName() { OptionRule datasourceMetadataFieldsByDataSourceName = - KAFKA_DATA_SOURCE_CHANNEL.getDatasourceMetadataFieldsByDataSourceName( - KAFKA_PLUGIN_NAME); + KAFKA_DATA_SOURCE_CHANNEL.getDatasourceMetadataFieldsByDataSourceName( + KAFKA_PLUGIN_NAME); Assertions.assertEquals( - 2, datasourceMetadataFieldsByDataSourceName.getOptionalOptions().size()); + 2, datasourceMetadataFieldsByDataSourceName.getOptionalOptions().size()); } @Test public void getTables() { List tables = - KAFKA_DATA_SOURCE_CHANNEL.getTables(KAFKA_PLUGIN_NAME, REQUEST_PARAMS, null); + KAFKA_DATA_SOURCE_CHANNEL.getTables(KAFKA_PLUGIN_NAME, REQUEST_PARAMS, null); log.info("{}", tables); Assertions.assertNotNull(tables); } @@ -74,7 +75,7 @@ public void getTables() { @Test public void getDatabases() { List databases = - KAFKA_DATA_SOURCE_CHANNEL.getDatabases(KAFKA_PLUGIN_NAME, REQUEST_PARAMS); + KAFKA_DATA_SOURCE_CHANNEL.getDatabases(KAFKA_PLUGIN_NAME, REQUEST_PARAMS); log.info("{}", databases); Assertions.assertNotNull(databases); } @@ -82,15 +83,15 @@ public void getDatabases() { @Test public void checkDataSourceConnectivity() { boolean dataSourceConnectivity = - KAFKA_DATA_SOURCE_CHANNEL.checkDataSourceConnectivity( - KAFKA_PLUGIN_NAME, REQUEST_PARAMS); + KAFKA_DATA_SOURCE_CHANNEL.checkDataSourceConnectivity( + KAFKA_PLUGIN_NAME, REQUEST_PARAMS); Assertions.assertTrue(dataSourceConnectivity); } @Test public void getTableFields() { List tableFields = - KAFKA_DATA_SOURCE_CHANNEL.getTableFields(KAFKA_PLUGIN_NAME, REQUEST_PARAMS, "", ""); + KAFKA_DATA_SOURCE_CHANNEL.getTableFields(KAFKA_PLUGIN_NAME, REQUEST_PARAMS, "", ""); log.info("{}", tableFields); Assertions.assertTrue(tableFields.isEmpty()); } @@ -98,8 +99,8 @@ public void getTableFields() { @Test public void testGetTableFields() { Map> tableFields = - KAFKA_DATA_SOURCE_CHANNEL.getTableFields( - KAFKA_PLUGIN_NAME, REQUEST_PARAMS, "", Collections.emptyList()); + KAFKA_DATA_SOURCE_CHANNEL.getTableFields( + KAFKA_PLUGIN_NAME, REQUEST_PARAMS, "", Collections.emptyList()); log.info("{}", tableFields); Assertions.assertTrue(tableFields.isEmpty()); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtilsTest.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtilsTest.java index 67d8d33f8..2ce0842fa 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtilsTest.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtilsTest.java @@ -17,10 +17,11 @@ package org.apache.seatunnel.datasource.plugin.kafka; -import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import com.google.common.collect.ImmutableMap; + import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -30,37 +31,41 @@ class KafkaRequestParamsUtilsTest { @Test void parsePropertiesFromRequestParams() { Map requestParams = - new ImmutableMap.Builder() - .put(KafkaOptionRule.BOOTSTRAP_SERVERS.key(), "localhost:9092") - .put( - KafkaOptionRule.KAFKA_CONFIG.key(), - "{" + "security.protocol = SASL_PLAINTEXT" + "}") - .build(); + new ImmutableMap.Builder() + .put(KafkaOptionRule.BOOTSTRAP_SERVERS.key(), "localhost:9092") + .put( + KafkaOptionRule.KAFKA_CONFIG.key(), + "{" + "security.protocol = SASL_PLAINTEXT" + "}") + .build(); Properties properties = - KafkaRequestParamsUtils.parsePropertiesFromRequestParams(requestParams); + KafkaRequestParamsUtils.parsePropertiesFromRequestParams(requestParams); Assertions.assertEquals("SASL_PLAINTEXT", properties.getProperty("security.protocol")); } @Test void parsePropertiesFromRequestParamsBadCase() { Assertions.assertDoesNotThrow( - () -> - KafkaRequestParamsUtils.parsePropertiesFromRequestParams( - new ImmutableMap.Builder() - .put(KafkaOptionRule.BOOTSTRAP_SERVERS.key(), "localhost:9092") - .put(KafkaOptionRule.KAFKA_CONFIG.key(), "{}") - .build())); + () -> + KafkaRequestParamsUtils.parsePropertiesFromRequestParams( + new ImmutableMap.Builder() + .put( + KafkaOptionRule.BOOTSTRAP_SERVERS.key(), + "localhost:9092") + .put(KafkaOptionRule.KAFKA_CONFIG.key(), "{}") + .build())); Assertions.assertThrows( - IllegalArgumentException.class, - () -> KafkaRequestParamsUtils.parsePropertiesFromRequestParams(new HashMap<>())); + IllegalArgumentException.class, + () -> KafkaRequestParamsUtils.parsePropertiesFromRequestParams(new HashMap<>())); Assertions.assertDoesNotThrow( - () -> - KafkaRequestParamsUtils.parsePropertiesFromRequestParams( - new ImmutableMap.Builder() - .put(KafkaOptionRule.BOOTSTRAP_SERVERS.key(), "localhost:9092") - .put(KafkaOptionRule.KAFKA_CONFIG.key(), "") - .build())); + () -> + KafkaRequestParamsUtils.parsePropertiesFromRequestParams( + new ImmutableMap.Builder() + .put( + KafkaOptionRule.BOOTSTRAP_SERVERS.key(), + "localhost:9092") + .put(KafkaOptionRule.KAFKA_CONFIG.key(), "") + .build())); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java index 07a9f3859..dd26ac14d 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java @@ -22,9 +22,10 @@ import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; +import org.apache.commons.lang3.StringUtils; + import com.google.common.collect.Sets; import lombok.NonNull; -import org.apache.commons.lang3.StringUtils; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -42,7 +43,7 @@ public class MysqlCDCDataSourceChannel implements DataSourceChannel { public static final Set MYSQL_SYSTEM_DATABASES = - Sets.newHashSet("information_schema", "mysql", "performance_schema", "sys"); + Sets.newHashSet("information_schema", "mysql", "performance_schema", "sys"); @Override public boolean canAbleGetSchema() { @@ -61,7 +62,7 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - String pluginName, Map requestParams, String database) { + String pluginName, Map requestParams, String database) { return this.getTableNames(requestParams, database); } @@ -76,22 +77,22 @@ public List getDatabases(String pluginName, Map requestP @Override public boolean checkDataSourceConnectivity( - String pluginName, Map requestParams) { + String pluginName, Map requestParams) { return this.checkJdbcConnectivity(requestParams); } @Override public List getTableFields( - String pluginName, Map requestParams, String database, String table) { + String pluginName, Map requestParams, String database, String table) { return getTableFields(requestParams, database, table); } @Override public Map> getTableFields( - String pluginName, - Map requestParams, - String database, - List tables) { + String pluginName, + Map requestParams, + String database, + List tables) { Map> tableFields = new HashMap<>(tables.size()); for (String table : tables) { tableFields.put(table, getTableFields(requestParams, database, table)); @@ -102,9 +103,9 @@ public Map> getTableFields( @SuppressWarnings("checkstyle:MagicNumber") protected boolean checkJdbcConnectivity(Map requestParams) { try (Connection connection = init(requestParams); - Statement statement = connection.createStatement()) { + Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery("SHOW MASTER STATUS");) { + try (ResultSet resultSet = statement.executeQuery("SHOW MASTER STATUS"); ) { if (resultSet.next()) { String binlogFile = resultSet.getString("File"); if (StringUtils.isBlank(binlogFile)) { @@ -116,7 +117,7 @@ protected boolean checkJdbcConnectivity(Map requestParams) { } try (ResultSet resultSet = - statement.executeQuery("SHOW VARIABLES LIKE 'binlog_format'")) { + statement.executeQuery("SHOW VARIABLES LIKE 'binlog_format'")) { if (resultSet.next()) { String binlogFormat = resultSet.getString("Value"); if (!"ROW".equalsIgnoreCase(binlogFormat)) { @@ -128,7 +129,7 @@ protected boolean checkJdbcConnectivity(Map requestParams) { } try (ResultSet resultSet = - statement.executeQuery("SHOW VARIABLES LIKE 'binlog_row_image'")) { + statement.executeQuery("SHOW VARIABLES LIKE 'binlog_row_image'")) { if (resultSet.next()) { String binlogRowImage = resultSet.getString("Value"); if (!"FULL".equalsIgnoreCase(binlogRowImage)) { @@ -141,7 +142,7 @@ protected boolean checkJdbcConnectivity(Map requestParams) { return true; } catch (Exception e) { throw new DataSourcePluginException( - "check jdbc connectivity failed, " + e.getMessage(), e); + "check jdbc connectivity failed, " + e.getMessage(), e); } } @@ -151,7 +152,7 @@ protected Connection init(Map requestParams) throws SQLException } String url = requestParams.get(MysqlCDCOptionRule.BASE_URL.key()); if (null != requestParams.get(MysqlCDCOptionRule.PASSWORD.key()) - && null != requestParams.get(MysqlCDCOptionRule.USERNAME.key())) { + && null != requestParams.get(MysqlCDCOptionRule.USERNAME.key())) { String username = requestParams.get(MysqlCDCOptionRule.USERNAME.key()); String password = requestParams.get(MysqlCDCOptionRule.PASSWORD.key()); return DriverManager.getConnection(url, username, password); @@ -162,8 +163,8 @@ protected Connection init(Map requestParams) throws SQLException protected List getDataBaseNames(Map requestParams) throws SQLException { List dbNames = new ArrayList<>(); try (Connection connection = init(requestParams); - PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); - ResultSet re = statement.executeQuery()) { + PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); + ResultSet re = statement.executeQuery()) { // filter system databases while (re.next()) { String dbName = re.getString("database"); @@ -178,10 +179,10 @@ protected List getDataBaseNames(Map requestParams) throw protected List getTableNames(Map requestParams, String dbName) { List tableNames = new ArrayList<>(); try (Connection connection = init(requestParams); - ResultSet resultSet = - connection - .getMetaData() - .getTables(dbName, null, null, new String[]{"TABLE"})) { + ResultSet resultSet = + connection + .getMetaData() + .getTables(dbName, null, null, new String[] {"TABLE"})) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); if (StringUtils.isNotBlank(tableName)) { @@ -195,9 +196,9 @@ protected List getTableNames(Map requestParams, String d } protected List getTableFields( - Map requestParams, String dbName, String tableName) { + Map requestParams, String dbName, String tableName) { List tableFields = new ArrayList<>(); - try (Connection connection = init(requestParams);) { + try (Connection connection = init(requestParams); ) { DatabaseMetaData metaData = connection.getMetaData(); String primaryKey = getPrimaryKey(metaData, dbName, tableName); ResultSet resultSet = metaData.getColumns(dbName, null, tableName, null); @@ -223,7 +224,7 @@ protected List getTableFields( } private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { + throws SQLException { ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); while (primaryKeysInfo.next()) { return primaryKeysInfo.getString("COLUMN_NAME"); @@ -233,8 +234,8 @@ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String ta private boolean isNotSystemDatabase(String dbName) { return MYSQL_SYSTEM_DATABASES.stream() - .noneMatch( - systemDatabase -> StringUtils.equalsAnyIgnoreCase(systemDatabase, dbName)); + .noneMatch( + systemDatabase -> StringUtils.equalsAnyIgnoreCase(systemDatabase, dbName)); } private boolean convertToBoolean(Object value) { diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourcePluginInfo.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourcePluginInfo.java index d496945de..3c47d5a51 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourcePluginInfo.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourcePluginInfo.java @@ -17,11 +17,11 @@ package org.apache.seatunnel.datasource.plugin.api; -import static com.google.common.base.Preconditions.checkNotNull; - import lombok.Builder; import lombok.Data; +import static com.google.common.base.Preconditions.checkNotNull; + @Data @Builder public class DataSourcePluginInfo { @@ -32,18 +32,14 @@ public class DataSourcePluginInfo { public String version; - /** - * @see DatasourcePluginTypeEnum - */ + /** @see DatasourcePluginTypeEnum */ private Integer type; - /** - * whether support virtual tables, default false - */ + /** whether support virtual tables, default false */ private Boolean supportVirtualTables; public DataSourcePluginInfo( - String name, String icon, String version, Integer type, Boolean supportVirtualTables) { + String name, String icon, String version, Integer type, Boolean supportVirtualTables) { this.name = checkNotNull(name, "name can not be null"); this.icon = checkNotNull(icon, "icon can not be null"); this.version = checkNotNull(version, "version can not be null"); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/common/ParamtersUtils.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/common/ParamtersUtils.java index 2a27a5684..257d9f3d8 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/common/ParamtersUtils.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/common/ParamtersUtils.java @@ -23,10 +23,10 @@ public class ParamtersUtils { /** - * for some parameters, we need to convert them to {@link Map} eg: s3Options { - * "access.value": "org.apache.hadoop.fs.s3a.S3AFileSystem", "access.key": - * "AKIAIOSFODNN7EXAMPLE", "hadoop_s3_properties": " fs.s3a.impl = - * org.apache.hadoop.fs.s3a.S3AFileSystem fs.s3a.access.key = AKIAIOSFODNN7EXAMPLE " + * for some parameters, we need to convert them to {@link Map} eg: s3Options { "access.value": + * "org.apache.hadoop.fs.s3a.S3AFileSystem", "access.key": "AKIAIOSFODNN7EXAMPLE", + * "hadoop_s3_properties": " fs.s3a.impl = org.apache.hadoop.fs.s3a.S3AFileSystem + * fs.s3a.access.key = AKIAIOSFODNN7EXAMPLE " * *

Convert parameters to {@link Map} * diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java index f32561867..8da509c12 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java @@ -17,15 +17,16 @@ package org.apache.seatunnel.datasource.plugin.redshift.s3; -import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; - -import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import lombok.extern.slf4j.Slf4j; + import java.util.Arrays; import java.util.Map; +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; + @Slf4j public class HadoopS3AConfiguration { @@ -42,11 +43,11 @@ public static Configuration getConfiguration(Map s3Options) { if (!s3Options.containsKey(S3RedshiftOptionRule.BUCKET.key())) { throw new IllegalArgumentException( - "S3Redshift datasource bucket is null, please check your config"); + "S3Redshift datasource bucket is null, please check your config"); } if (!s3Options.containsKey(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key())) { throw new IllegalArgumentException( - "S3Redshift datasource endpoint is null, please check your config"); + "S3Redshift datasource endpoint is null, please check your config"); } String bucket = s3Options.get(S3RedshiftOptionRule.BUCKET.key()); @@ -58,38 +59,38 @@ public static Configuration getConfiguration(Map s3Options) { Configuration hadoopConf = new Configuration(); hadoopConf.set(FS_DEFAULT_NAME_KEY, bucket); hadoopConf.set( - S3RedshiftOptionRule.FS_S3A_ENDPOINT.key(), - s3Options.get(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key())); + S3RedshiftOptionRule.FS_S3A_ENDPOINT.key(), + s3Options.get(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key())); hadoopConf.set(formatKey(protocol, HDFS_IMPL_KEY), fsImpl); if (s3Options.containsKey(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key())) { Arrays.stream( - s3Options - .get(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key()) - .split("\n")) - .map(String::trim) - .filter(StringUtils::isNotBlank) - .forEach( - line -> { - String[] kv = line.split("="); - if (kv.length == 2) { - hadoopConf.set(kv[0].trim(), kv[1].trim()); - } - }); + s3Options + .get(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key()) + .split("\n")) + .map(String::trim) + .filter(StringUtils::isNotBlank) + .forEach( + line -> { + String[] kv = line.split("="); + if (kv.length == 2) { + hadoopConf.set(kv[0].trim(), kv[1].trim()); + } + }); } if (S3RedshiftOptionRule.S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider - .getProvider() - .equals(s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()))) { + .getProvider() + .equals(s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()))) { hadoopConf.set( - S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(), - s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key())); + S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(), + s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key())); hadoopConf.set( - "fs.s3a.access.key", s3Options.get(S3RedshiftOptionRule.ACCESS_KEY.key())); + "fs.s3a.access.key", s3Options.get(S3RedshiftOptionRule.ACCESS_KEY.key())); hadoopConf.set( - "fs.s3a.secret.key", s3Options.get(S3RedshiftOptionRule.SECRET_KEY.key())); + "fs.s3a.secret.key", s3Options.get(S3RedshiftOptionRule.SECRET_KEY.key())); } else { hadoopConf.set( - S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(), - s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key())); + S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(), + s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key())); } return hadoopConf; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java index 07aceb327..de9e2b09e 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java @@ -23,13 +23,14 @@ import org.apache.seatunnel.datasource.plugin.api.model.TableField; import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; -import com.google.common.collect.Sets; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import com.google.common.collect.Sets; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; @@ -59,13 +60,13 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { return getTableNames(requestParams, database); } @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try { return getDataBaseNames(pluginName, requestParams); } catch (SQLException e) { @@ -75,7 +76,7 @@ public List getDatabases( @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { checkHdfsS3Connection(requestParams); checkJdbcConnection(requestParams); return true; @@ -83,19 +84,19 @@ public boolean checkDataSourceConnectivity( @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { return getTableFields(requestParams, database, table); } @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { // not need this method return null; } @@ -113,14 +114,14 @@ private void checkJdbcConnection(Map requestParams) { return; } catch (SQLException e) { throw new DataSourcePluginException( - "Check Redshift jdbc connection failed,please check your config", e); + "Check Redshift jdbc connection failed,please check your config", e); } } try (Connection ignored = DriverManager.getConnection(jdbcUrl, username, password)) { log.info("Redshift jdbc connection is valid"); } catch (SQLException e) { throw new DataSourcePluginException( - "Check Redshift jdbc connection failed,please check your config", e); + "Check Redshift jdbc connection failed,please check your config", e); } } @@ -130,20 +131,20 @@ private void checkHdfsS3Connection(Map requestParams) { fs.getFileStatus(new org.apache.hadoop.fs.Path("/")); } catch (IOException e) { throw new DataSourcePluginException( - "S3 configuration is invalid, please check your config", e); + "S3 configuration is invalid, please check your config", e); } } protected Connection init(Map requestParams, String databaseName) - throws SQLException { + throws SQLException { if (null == requestParams.get(S3RedshiftOptionRule.JDBC_URL.key())) { throw new DataSourcePluginException("Jdbc url is null"); } String url = - JdbcUtils.replaceDatabase( - requestParams.get(S3RedshiftOptionRule.JDBC_URL.key()), databaseName); + JdbcUtils.replaceDatabase( + requestParams.get(S3RedshiftOptionRule.JDBC_URL.key()), databaseName); if (null != requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key()) - && null != requestParams.get(S3RedshiftOptionRule.JDBC_USER.key())) { + && null != requestParams.get(S3RedshiftOptionRule.JDBC_USER.key())) { String username = requestParams.get(S3RedshiftOptionRule.JDBC_USER.key()); String password = requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key()); return DriverManager.getConnection(url, username, password); @@ -152,12 +153,12 @@ protected Connection init(Map requestParams, String databaseName } protected List getDataBaseNames(String pluginName, Map requestParams) - throws SQLException { + throws SQLException { List dbNames = new ArrayList<>(); try (Connection connection = init(requestParams, null); - PreparedStatement statement = - connection.prepareStatement("select datname from pg_database;"); - ResultSet re = statement.executeQuery()) { + PreparedStatement statement = + connection.prepareStatement("select datname from pg_database;"); + ResultSet re = statement.executeQuery()) { while (re.next()) { String dbName = re.getString("datname"); if (StringUtils.isNotBlank(dbName) && isNotSystemDatabase(dbName)) { @@ -172,9 +173,9 @@ protected List getDataBaseNames(String pluginName, Map r protected List getTableNames(Map requestParams, String dbName) { List tableNames = new ArrayList<>(); - try (Connection connection = init(requestParams, dbName);) { + try (Connection connection = init(requestParams, dbName); ) { ResultSet resultSet = - connection.getMetaData().getTables(dbName, null, null, new String[]{"TABLE"}); + connection.getMetaData().getTables(dbName, null, null, new String[] {"TABLE"}); while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); if (StringUtils.isNotBlank(tableName)) { @@ -188,15 +189,15 @@ protected List getTableNames(Map requestParams, String d } protected List getTableFields( - Map requestParams, String dbName, String tableName) { + Map requestParams, String dbName, String tableName) { List tableFields = new ArrayList<>(); - try (Connection connection = init(requestParams, dbName);) { + try (Connection connection = init(requestParams, dbName); ) { DatabaseMetaData metaData = connection.getMetaData(); String primaryKey = getPrimaryKey(metaData, dbName, tableName); String[] split = tableName.split("\\."); if (split.length != 2) { throw new DataSourcePluginException( - "Postgresql tableName should composed by schemaName.tableName"); + "Postgresql tableName should composed by schemaName.tableName"); } ResultSet resultSet = metaData.getColumns(dbName, split[0], split[1], null); while (resultSet.next()) { @@ -221,7 +222,7 @@ protected List getTableFields( } private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { + throws SQLException { ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); while (primaryKeysInfo.next()) { return primaryKeysInfo.getString("COLUMN_NAME"); @@ -255,14 +256,14 @@ private boolean convertToBoolean(Object value) { } public static final Set POSTGRESQL_SYSTEM_DATABASES = - Sets.newHashSet( - "information_schema", - "pg_catalog", - "root", - "pg_toast", - "pg_temp_1", - "pg_toast_temp_1", - "postgres", - "template0", - "template1"); + Sets.newHashSet( + "information_schema", + "pg_catalog", + "root", + "pg_toast", + "pg_temp_1", + "pg_toast_temp_1", + "postgres", + "template0", + "template1"); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java index 3ef212acd..ee6aa4ba0 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java @@ -37,14 +37,14 @@ public String factoryIdentifier() { @Override public Set supportedDataSources() { DataSourcePluginInfo s3DatasourcePluginInfo = - DataSourcePluginInfo.builder() - .name("S3-Redshift") - .type(DatasourcePluginTypeEnum.DATABASE.getCode()) - .version("1.0.0") - .supportVirtualTables(false) - .icon("S3-Redshift") - .icon("S3-Redshift") - .build(); + DataSourcePluginInfo.builder() + .name("S3-Redshift") + .type(DatasourcePluginTypeEnum.DATABASE.getCode()) + .version("1.0.0") + .supportVirtualTables(false) + .icon("S3-Redshift") + .icon("S3-Redshift") + .build(); return Sets.newHashSet(s3DatasourcePluginInfo); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/HadoopS3AConfiguration.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/HadoopS3AConfiguration.java index ecce9793c..3c18b2965 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/HadoopS3AConfiguration.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/HadoopS3AConfiguration.java @@ -17,16 +17,17 @@ package org.apache.seatunnel.datasource.plugin.s3; -import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; - import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; -import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; +import lombok.extern.slf4j.Slf4j; + import java.util.Map; +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; + @Slf4j public class HadoopS3AConfiguration { @@ -43,11 +44,11 @@ public static Configuration getConfiguration(Map s3Options) { if (!s3Options.containsKey(S3OptionRule.BUCKET.key())) { throw new IllegalArgumentException( - "S3 datasource bucket is null, please check your config"); + "S3 datasource bucket is null, please check your config"); } if (!s3Options.containsKey(S3OptionRule.FS_S3A_ENDPOINT.key())) { throw new IllegalArgumentException( - "S3 datasource endpoint is null, please check your config"); + "S3 datasource endpoint is null, please check your config"); } String bucket = s3Options.get(S3OptionRule.BUCKET.key()); @@ -59,32 +60,32 @@ public static Configuration getConfiguration(Map s3Options) { Configuration hadoopConf = new Configuration(); hadoopConf.set(FS_DEFAULT_NAME_KEY, bucket); hadoopConf.set( - S3OptionRule.FS_S3A_ENDPOINT.key(), - s3Options.get(S3OptionRule.FS_S3A_ENDPOINT.key())); + S3OptionRule.FS_S3A_ENDPOINT.key(), + s3Options.get(S3OptionRule.FS_S3A_ENDPOINT.key())); hadoopConf.set(formatKey(protocol, HDFS_IMPL_KEY), fsImpl); if (s3Options.containsKey(S3OptionRule.HADOOP_S3_PROPERTIES.key())) { Config configObject = - ConfigFactory.parseString( - s3Options.get(S3OptionRule.HADOOP_S3_PROPERTIES.key())); + ConfigFactory.parseString( + s3Options.get(S3OptionRule.HADOOP_S3_PROPERTIES.key())); configObject - .entrySet() - .forEach( - entry -> { - hadoopConf.set( - entry.getKey(), entry.getValue().unwrapped().toString()); - }); + .entrySet() + .forEach( + entry -> { + hadoopConf.set( + entry.getKey(), entry.getValue().unwrapped().toString()); + }); } if (S3OptionRule.S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider.getProvider() - .equals(s3Options.get(S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()))) { + .equals(s3Options.get(S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()))) { hadoopConf.set( - S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(), - s3Options.get(S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key())); + S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(), + s3Options.get(S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key())); hadoopConf.set("fs.s3a.access.key", s3Options.get(S3OptionRule.ACCESS_KEY.key())); hadoopConf.set("fs.s3a.secret.key", s3Options.get(S3OptionRule.SECRET_KEY.key())); } else { hadoopConf.set( - S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(), - s3Options.get(S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key())); + S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(), + s3Options.get(S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key())); } return hadoopConf; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DataSourceFactory.java index 640b6f16f..5b2236562 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DataSourceFactory.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DataSourceFactory.java @@ -40,13 +40,13 @@ public String factoryIdentifier() { @Override public Set supportedDataSources() { DataSourcePluginInfo s3DatasourcePluginInfo = - DataSourcePluginInfo.builder() - .name(PLUGIN_NAME) - .type(DatasourcePluginTypeEnum.FILE.getCode()) - .version("1.0.0") - .supportVirtualTables(false) - .icon("S3File") - .build(); + DataSourcePluginInfo.builder() + .name(PLUGIN_NAME) + .type(DatasourcePluginTypeEnum.FILE.getCode()) + .version("1.0.0") + .supportVirtualTables(false) + .icon("S3File") + .build(); return Sets.newHashSet(s3DatasourcePluginInfo); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DatasourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DatasourceChannel.java index e1ba8de5e..cf0ee45b9 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DatasourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DatasourceChannel.java @@ -22,11 +22,12 @@ import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; -import lombok.NonNull; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import lombok.NonNull; + import java.io.IOException; import java.util.List; import java.util.Map; @@ -44,46 +45,46 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { throw new UnsupportedOperationException("getTables is not supported for S3 datasource"); } @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { throw new UnsupportedOperationException("getDatabases is not supported for S3 datasource"); } @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { Configuration conf = HadoopS3AConfiguration.getConfiguration(requestParams); try (FileSystem fs = FileSystem.get(conf)) { fs.listStatus(new Path("/")); return true; } catch (IOException e) { throw new DataSourcePluginException( - String.format("check s3 connectivity failed, config is: %s", requestParams), e); + String.format("check s3 connectivity failed, config is: %s", requestParams), e); } } @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { throw new UnsupportedOperationException( - "getTableFields is not supported for S3 datasource"); + "getTableFields is not supported for S3 datasource"); } @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { throw new UnsupportedOperationException( - "getTableFields is not supported for S3 datasource"); + "getTableFields is not supported for S3 datasource"); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java index 4fed9948f..20a23fae0 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java @@ -22,11 +22,12 @@ import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + import com.google.common.collect.Sets; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -45,7 +46,7 @@ public class SqlServerCDCDataSourceChannel implements DataSourceChannel { public static final Set MYSQL_SYSTEM_DATABASES = - Sets.newHashSet("master", "tempdb", "model", "msdb"); + Sets.newHashSet("master", "tempdb", "model", "msdb"); @Override public boolean canAbleGetSchema() { @@ -64,7 +65,7 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - String pluginName, Map requestParams, String database) { + String pluginName, Map requestParams, String database) { return this.getTableNames(requestParams, database); } @@ -79,10 +80,10 @@ public List getDatabases(String pluginName, Map requestP @Override public boolean checkDataSourceConnectivity( - String pluginName, Map requestParams) { + String pluginName, Map requestParams) { try (Connection connection = init(requestParams); - PreparedStatement statement = connection.prepareStatement("SELECT 1"); - ResultSet rs = statement.executeQuery()) { + PreparedStatement statement = connection.prepareStatement("SELECT 1"); + ResultSet rs = statement.executeQuery()) { return rs.next(); } catch (SQLException e) { throw new DataSourcePluginException("connect datasource failed", e); @@ -91,17 +92,17 @@ public boolean checkDataSourceConnectivity( @Override public List getTableFields( - String pluginName, Map requestParams, String database, String table) { + String pluginName, Map requestParams, String database, String table) { Pair pair = parseSchemaAndTable(table); return getTableFields(requestParams, database, pair.getLeft(), pair.getRight()); } @Override public Map> getTableFields( - String pluginName, - Map requestParams, - String database, - List tables) { + String pluginName, + Map requestParams, + String database, + List tables) { Map> tableFields = new HashMap<>(tables.size()); for (String table : tables) { tableFields.put(table, getTableFields(pluginName, requestParams, database, table)); @@ -115,7 +116,7 @@ private Connection init(Map requestParams) throws SQLException { } String url = requestParams.get(SqlServerCDCOptionRule.BASE_URL.key()); if (null != requestParams.get(SqlServerCDCOptionRule.PASSWORD.key()) - && null != requestParams.get(SqlServerCDCOptionRule.USERNAME.key())) { + && null != requestParams.get(SqlServerCDCOptionRule.USERNAME.key())) { String username = requestParams.get(SqlServerCDCOptionRule.USERNAME.key()); String password = requestParams.get(SqlServerCDCOptionRule.PASSWORD.key()); return DriverManager.getConnection(url, username, password); @@ -126,10 +127,10 @@ private Connection init(Map requestParams) throws SQLException { private List getDataBaseNames(Map requestParams) throws SQLException { List dbNames = new ArrayList<>(); try (Connection connection = init(requestParams); - PreparedStatement statement = - connection.prepareStatement( - "SELECT NAME FROM SYS.DATABASES WHERE IS_CDC_ENABLED = 1;"); - ResultSet re = statement.executeQuery()) { + PreparedStatement statement = + connection.prepareStatement( + "SELECT NAME FROM SYS.DATABASES WHERE IS_CDC_ENABLED = 1;"); + ResultSet re = statement.executeQuery()) { // filter system databases while (re.next()) { String dbName = re.getString("NAME"); @@ -144,18 +145,18 @@ private List getDataBaseNames(Map requestParams) throws private List getTableNames(Map requestParams, String dbName) { final String sql = - String.format( - "SELECT SCHEMAS.NAME AS SCHEMA_NAME, TABLES.NAME AS TABLE_NAME" - + " FROM %s.SYS.SCHEMAS AS SCHEMAS" - + " JOIN %s.SYS.TABLES AS TABLES" - + " ON SCHEMAS.SCHEMA_ID = TABLES.SCHEMA_ID" - + " AND TABLES.IS_TRACKED_BY_CDC = 1", - dbName, dbName); + String.format( + "SELECT SCHEMAS.NAME AS SCHEMA_NAME, TABLES.NAME AS TABLE_NAME" + + " FROM %s.SYS.SCHEMAS AS SCHEMAS" + + " JOIN %s.SYS.TABLES AS TABLES" + + " ON SCHEMAS.SCHEMA_ID = TABLES.SCHEMA_ID" + + " AND TABLES.IS_TRACKED_BY_CDC = 1", + dbName, dbName); List tableNames = new ArrayList<>(); try (Connection connection = init(requestParams); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(sql)) { + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { while (resultSet.next()) { String schemaName = resultSet.getString("SCHEMA_NAME"); String tableName = resultSet.getString("TABLE_NAME"); @@ -168,9 +169,9 @@ private List getTableNames(Map requestParams, String dbN } private List getTableFields( - Map requestParams, String dbName, String schemaName, String tableName) { + Map requestParams, String dbName, String schemaName, String tableName) { List tableFields = new ArrayList<>(); - try (Connection connection = init(requestParams);) { + try (Connection connection = init(requestParams); ) { DatabaseMetaData metaData = connection.getMetaData(); String primaryKey = getPrimaryKey(metaData, dbName, schemaName, tableName); ResultSet resultSet = metaData.getColumns(dbName, schemaName, tableName, null); @@ -196,8 +197,8 @@ private List getTableFields( } private String getPrimaryKey( - DatabaseMetaData metaData, String dbName, String schemaName, String tableName) - throws SQLException { + DatabaseMetaData metaData, String dbName, String schemaName, String tableName) + throws SQLException { ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, schemaName, tableName); while (primaryKeysInfo.next()) { return primaryKeysInfo.getString("COLUMN_NAME"); @@ -207,8 +208,8 @@ private String getPrimaryKey( private boolean isNotSystemDatabase(String dbName) { return MYSQL_SYSTEM_DATABASES.stream() - .noneMatch( - systemDatabase -> StringUtils.equalsAnyIgnoreCase(systemDatabase, dbName)); + .noneMatch( + systemDatabase -> StringUtils.equalsAnyIgnoreCase(systemDatabase, dbName)); } private boolean convertToBoolean(Object value) { diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceFactory.java index 321f9c8c6..bc4f276f1 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceFactory.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceFactory.java @@ -37,7 +37,7 @@ public String factoryIdentifier() { @Override public Set supportedDataSources() { return Collections.singleton( - SqlServerCDCDataSourceConfig.SQLSERVER_CDC_DATASOURCE_PLUGIN_INFO); + SqlServerCDCDataSourceConfig.SQLSERVER_CDC_DATASOURCE_PLUGIN_INFO); } @Override diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/test/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/test/TestSqlServerCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/test/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/test/TestSqlServerCDCDataSourceChannel.java index b815a3187..982c50721 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/test/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/test/TestSqlServerCDCDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/test/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/test/TestSqlServerCDCDataSourceChannel.java @@ -39,10 +39,17 @@ public void testConnect() { requestParams.put("username", "sa"); requestParams.put("password", "MyPass@word"); - for (String database : channel.getDatabases(SqlServerCDCDataSourceConfig.PLUGIN_NAME, requestParams)) { - final List tables = channel.getTables(SqlServerCDCDataSourceConfig.PLUGIN_NAME, requestParams, database); + for (String database : + channel.getDatabases(SqlServerCDCDataSourceConfig.PLUGIN_NAME, requestParams)) { + final List tables = + channel.getTables( + SqlServerCDCDataSourceConfig.PLUGIN_NAME, requestParams, database); final Map> tableFields = - channel.getTableFields(SqlServerCDCDataSourceConfig.PLUGIN_NAME, requestParams, database, tables); + channel.getTableFields( + SqlServerCDCDataSourceConfig.PLUGIN_NAME, + requestParams, + database, + tables); } } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksCatalog.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksCatalog.java index c04558b07..4acc43e60 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksCatalog.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksCatalog.java @@ -17,8 +17,6 @@ package org.apache.seatunnel.datasource.plugin.starrocks; -import static com.google.common.base.Preconditions.checkArgument; - import org.apache.seatunnel.api.table.catalog.PrimaryKey; import org.apache.seatunnel.api.table.catalog.TablePath; import org.apache.seatunnel.api.table.catalog.exception.CatalogException; @@ -27,6 +25,7 @@ import org.apache.seatunnel.datasource.plugin.api.model.TableField; import org.apache.commons.lang3.StringUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,6 +41,8 @@ import java.util.Optional; import java.util.Set; +import static com.google.common.base.Preconditions.checkArgument; + public class StarRocksCatalog { protected final String catalogName; @@ -79,8 +80,8 @@ public StarRocksCatalog(String catalogName, String username, String pwd, String public List listDatabases() throws CatalogException { List databases = new ArrayList<>(); try (Connection conn = DriverManager.getConnection(defaultUrl, username, pwd); - PreparedStatement ps = conn.prepareStatement("SHOW DATABASES;"); - ResultSet rs = ps.executeQuery();) { + PreparedStatement ps = conn.prepareStatement("SHOW DATABASES;"); + ResultSet rs = ps.executeQuery(); ) { while (rs.next()) { String databaseName = rs.getString(1); @@ -92,19 +93,19 @@ public List listDatabases() throws CatalogException { return databases; } catch (Exception e) { throw new CatalogException( - String.format("Failed listing database in catalog %s", this.catalogName), e); + String.format("Failed listing database in catalog %s", this.catalogName), e); } } public List listTables(String databaseName) - throws CatalogException, DatabaseNotExistException { + throws CatalogException, DatabaseNotExistException { if (!databaseExists(databaseName)) { throw new DatabaseNotExistException(this.catalogName, databaseName); } try (Connection conn = DriverManager.getConnection(baseUrl + databaseName, username, pwd); - PreparedStatement ps = conn.prepareStatement("SHOW TABLES;"); - ResultSet rs = ps.executeQuery()) { + PreparedStatement ps = conn.prepareStatement("SHOW TABLES;"); + ResultSet rs = ps.executeQuery()) { List tables = new ArrayList<>(); @@ -115,29 +116,29 @@ public List listTables(String databaseName) return tables; } catch (Exception e) { throw new CatalogException( - String.format("Failed listing database in catalog %s", catalogName), e); + String.format("Failed listing database in catalog %s", catalogName), e); } } public List getTable(TablePath tablePath) - throws CatalogException, TableNotExistException { + throws CatalogException, TableNotExistException { if (!tableExists(tablePath)) { throw new TableNotExistException(catalogName, tablePath); } String dbUrl = baseUrl + tablePath.getDatabaseName(); try (Connection conn = DriverManager.getConnection(dbUrl, username, pwd); - PreparedStatement statement = - conn.prepareStatement( - String.format( - "SELECT * FROM %s WHERE 1 = 0;", - String.format( - "`%s`.`%s`", - tablePath.getDatabaseName(), - tablePath.getTableName())));) { + PreparedStatement statement = + conn.prepareStatement( + String.format( + "SELECT * FROM %s WHERE 1 = 0;", + String.format( + "`%s`.`%s`", + tablePath.getDatabaseName(), + tablePath.getTableName()))); ) { Optional primaryKey = - getPrimaryKey(tablePath.getDatabaseName(), tablePath.getTableName()); + getPrimaryKey(tablePath.getDatabaseName(), tablePath.getTableName()); ResultSetMetaData tableMetaData = statement.getMetaData(); @@ -148,13 +149,13 @@ public List getTable(TablePath tablePath) tableField.setType(tableMetaData.getColumnTypeName(i)); tableField.setComment(tableMetaData.getColumnLabel(i)); tableField.setNullable( - tableMetaData.isNullable(i) == ResultSetMetaData.columnNullable); + tableMetaData.isNullable(i) == ResultSetMetaData.columnNullable); tableField.setPrimaryKey( - primaryKey.isPresent() - && primaryKey - .get() - .getColumnNames() - .contains(tableField.getName())); + primaryKey.isPresent() + && primaryKey + .get() + .getColumnNames() + .contains(tableField.getName())); // TODO add default value tableField.setDefaultValue(null); fields.add(tableField); @@ -162,7 +163,7 @@ public List getTable(TablePath tablePath) return fields; } catch (Exception e) { throw new CatalogException( - String.format("Failed getting table %s", tablePath.getFullName()), e); + String.format("Failed getting table %s", tablePath.getFullName()), e); } } @@ -178,12 +179,12 @@ protected Optional getPrimaryKey(String schema, String table) throws List pkFields = new ArrayList<>(); try (Connection conn = DriverManager.getConnection(defaultUrl, username, pwd); - PreparedStatement statement = - conn.prepareStatement( - String.format( - "SELECT COLUMN_NAME FROM information_schema.columns where TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' AND COLUMN_KEY = 'PRI' ORDER BY ORDINAL_POSITION", - schema, table)); - ResultSet rs = statement.executeQuery()) { + PreparedStatement statement = + conn.prepareStatement( + String.format( + "SELECT COLUMN_NAME FROM information_schema.columns where TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' AND COLUMN_KEY = 'PRI' ORDER BY ORDINAL_POSITION", + schema, table)); + ResultSet rs = statement.executeQuery()) { while (rs.next()) { String columnName = rs.getString("COLUMN_NAME"); pkFields.add(columnName); @@ -216,7 +217,7 @@ public static boolean validateJdbcUrlWithDatabase(String url) { public boolean tableExists(TablePath tablePath) throws CatalogException { try { return databaseExists(tablePath.getDatabaseName()) - && listTables(tablePath.getDatabaseName()).contains(tablePath.getTableName()); + && listTables(tablePath.getDatabaseName()).contains(tablePath.getTableName()); } catch (DatabaseNotExistException e) { return false; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksDataSourceChannel.java index a5ed265f4..835864a4f 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksDataSourceChannel.java @@ -17,18 +17,19 @@ package org.apache.seatunnel.datasource.plugin.starrocks; +import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper; + import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.catalog.TablePath; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper; - -import lombok.NonNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import lombok.NonNull; + import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; @@ -59,21 +60,21 @@ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pl @Override public List getTables( - @NonNull String pluginName, Map requestParams, String database) { + @NonNull String pluginName, Map requestParams, String database) { StarRocksCatalog catalog = getCatalog(requestParams); return catalog.listTables(database); } @Override public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { StarRocksCatalog catalog = getCatalog(requestParams); return catalog.listDatabases(); } @Override public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { + @NonNull String pluginName, @NonNull Map requestParams) { try { StarRocksCatalog catalog = getCatalog(requestParams); String nodeUrls = requestParams.get(StarRocksOptionRule.NODE_URLS.key()); @@ -85,7 +86,7 @@ public boolean checkDataSourceConnectivity( return true; } catch (Exception e) { throw new DataSourcePluginException( - "check StarRocks connectivity failed, " + e.getMessage(), e); + "check StarRocks connectivity failed, " + e.getMessage(), e); } } @@ -96,7 +97,7 @@ private static boolean telnet(String nodeUrl) throws IOException { try { String[] hostAndPort = nodeUrl.split(":"); socket.connect( - new InetSocketAddress(hostAndPort[0], Integer.parseInt(hostAndPort[1])), 1000); + new InetSocketAddress(hostAndPort[0], Integer.parseInt(hostAndPort[1])), 1000); isConnected = socket.isConnected(); } catch (IOException e) { LOGGER.error("telnet error", e); @@ -113,24 +114,24 @@ private static boolean telnet(String nodeUrl) throws IOException { @Override public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { StarRocksCatalog catalog = getCatalog(requestParams); return catalog.getTable(TablePath.of(database, table)); } @Override public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { StarRocksCatalog catalog = getCatalog(requestParams); Map> tableFields = new HashMap<>(); tables.forEach( - table -> tableFields.put(table, catalog.getTable(TablePath.of(database, table)))); + table -> tableFields.put(table, catalog.getTable(TablePath.of(database, table)))); return tableFields; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml index 91cdc7409..2336b7b0a 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. --> - 4.0.0 diff --git a/seatunnel-server/pom.xml b/seatunnel-server/pom.xml index 7bfbf9d67..0a623719c 100644 --- a/seatunnel-server/pom.xml +++ b/seatunnel-server/pom.xml @@ -13,15 +13,14 @@ See the License for the specific language governing permissions and limitations under the License. --> - + 4.0.0 - seatunnel-web org.apache.seatunnel + seatunnel-web ${revision} - 4.0.0 seatunnel-server pom @@ -32,4 +31,4 @@ seatunnel-scheduler seatunnel-server-common - \ No newline at end of file + diff --git a/seatunnel-server/seatunnel-app/pom.xml b/seatunnel-server/seatunnel-app/pom.xml index ca908b06a..6c5da47b3 100644 --- a/seatunnel-server/seatunnel-app/pom.xml +++ b/seatunnel-server/seatunnel-app/pom.xml @@ -13,15 +13,14 @@ See the License for the specific language governing permissions and limitations under the License. --> - + 4.0.0 - seatunnel-server org.apache.seatunnel + seatunnel-server ${revision} - 4.0.0 seatunnel-app @@ -83,8 +82,8 @@ spring-boot-starter-tomcat - log4j-to-slf4j org.apache.logging.log4j + log4j-to-slf4j @@ -125,16 +124,16 @@ ${springfox-swagger.version} - spring-aop org.springframework + spring-aop - spring-beans org.springframework + spring-beans - spring-context org.springframework + spring-context com.fasterxml.jackson.core @@ -158,8 +157,8 @@ hibernate-validator - classmate com.fasterxml + classmate @@ -188,8 +187,8 @@ ${project.version} - slf4j-log4j12 org.slf4j + slf4j-log4j12 @@ -200,8 +199,8 @@ ${project.version} - slf4j-log4j12 org.slf4j + slf4j-log4j12 @@ -240,4 +239,4 @@ cron-utils - \ No newline at end of file + diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java index 4947195fe..967167890 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java @@ -25,7 +25,8 @@ import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.transaction.annotation.EnableTransactionManagement; -@SpringBootApplication(scanBasePackages = {"org.apache.seatunnel.app", "org.apache.seatunnel.scheduler"}) +@SpringBootApplication( + scanBasePackages = {"org.apache.seatunnel.app", "org.apache.seatunnel.scheduler"}) @EnableTransactionManagement @EnableConfigurationProperties @EnableScheduling diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/adapter/SeatunnelWebAdapter.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/adapter/SeatunnelWebAdapter.java index 84117d8ef..a20f5a8f0 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/adapter/SeatunnelWebAdapter.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/adapter/SeatunnelWebAdapter.java @@ -48,8 +48,7 @@ public AuthenticationInterceptor authenticationInterceptor() { return new AuthenticationInterceptor(); } - @Resource - private UserIdMethodArgumentResolver currentUserMethodArgumentResolver; + @Resource private UserIdMethodArgumentResolver currentUserMethodArgumentResolver; /** * Cookie @@ -70,10 +69,18 @@ public LocaleResolver localeResolver() { @Override public void addInterceptors(InterceptorRegistry registry) { registry.addInterceptor(authenticationInterceptor()) - .order(1) - .addPathPatterns(LOGIN_INTERCEPTOR_PATH_PATTERN) - .excludePathPatterns(LOGIN_PATH_PATTERN, REGISTER_PATH_PATTERN, - "/swagger-resources/**", "/webjars/**", "/v2/**", "*.html", "/ui/**", "/error", "/swagger-ui.html**"); + .order(1) + .addPathPatterns(LOGIN_INTERCEPTOR_PATH_PATTERN) + .excludePathPatterns( + LOGIN_PATH_PATTERN, + REGISTER_PATH_PATTERN, + "/swagger-resources/**", + "/webjars/**", + "/v2/**", + "*.html", + "/ui/**", + "/error", + "/swagger-ui.html**"); } @Override @@ -84,7 +91,8 @@ public void addArgumentResolvers(List argumentRes @Override public void addResourceHandlers(ResourceHandlerRegistry registry) { registry.addResourceHandler("/static/**").addResourceLocations("classpath:/static/"); - registry.addResourceHandler("/webjars/**").addResourceLocations("classpath:/META-INF/resources/webjars/"); + registry.addResourceHandler("/webjars/**") + .addResourceLocations("classpath:/META-INF/resources/webjars/"); registry.addResourceHandler("/ui/**").addResourceLocations("file:ui/"); } @@ -93,5 +101,4 @@ public void addViewControllers(ViewControllerRegistry registry) { registry.addViewController("/").setViewName("redirect:/ui/"); registry.addViewController("/ui/").setViewName("forward:/ui/index.html"); } - } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/aspect/LogoutAspect.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/aspect/LogoutAspect.java index 3d481c2f5..48c7b6769 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/aspect/LogoutAspect.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/aspect/LogoutAspect.java @@ -17,11 +17,8 @@ package org.apache.seatunnel.app.aspect; -import static org.apache.seatunnel.server.common.Constants.USER_ID; - import org.apache.seatunnel.app.dal.dao.IUserDao; -import lombok.extern.slf4j.Slf4j; import org.aspectj.lang.JoinPoint; import org.aspectj.lang.annotation.Aspect; import org.aspectj.lang.annotation.Before; @@ -31,26 +28,28 @@ import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; +import lombok.extern.slf4j.Slf4j; + import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; +import static org.apache.seatunnel.server.common.Constants.USER_ID; + @Slf4j @Aspect @Component @Order(2) public class LogoutAspect { - @Resource - private IUserDao userDaoImpl; + @Resource private IUserDao userDaoImpl; @Pointcut("execution(public * org.apache.seatunnel.app.controller.UserController.logout(..))") - public void logoutPointCut() { - - } + public void logoutPointCut() {} @Before("logoutPointCut()") public void check(JoinPoint pjp) { - ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); + ServletRequestAttributes attributes = + (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); HttpServletRequest request = attributes.getRequest(); final Integer userId = (Integer) request.getAttribute(USER_ID); userDaoImpl.disableToken(userId); diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/aspect/UserId.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/aspect/UserId.java index 628e103ca..8d812be39 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/aspect/UserId.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/aspect/UserId.java @@ -24,5 +24,4 @@ @Target({ElementType.PARAMETER}) @Retention(RetentionPolicy.RUNTIME) -public @interface UserId { -} +public @interface UserId {} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/RoleTypeEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/RoleTypeEnum.java index cd2566abc..836bfe7e4 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/RoleTypeEnum.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/RoleTypeEnum.java @@ -23,7 +23,7 @@ public enum RoleTypeEnum { ; private final int code; - private final String description; + private final String description; RoleTypeEnum(int code, String description) { this.code = code; diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptParamStatusEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptParamStatusEnum.java index 56195dd8d..22f286be2 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptParamStatusEnum.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptParamStatusEnum.java @@ -23,7 +23,7 @@ public enum ScriptParamStatusEnum { ; private final int code; - private final String description; + private final String description; ScriptParamStatusEnum(int code, String description) { this.code = code; diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptStatusEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptStatusEnum.java index 0a4144e04..154fd2a5b 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptStatusEnum.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptStatusEnum.java @@ -24,7 +24,7 @@ public enum ScriptStatusEnum { ; private final int code; - private final String description; + private final String description; ScriptStatusEnum(int code, String description) { this.code = code; diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptTypeEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptTypeEnum.java index 89d627c60..353f2b2c3 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptTypeEnum.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptTypeEnum.java @@ -23,7 +23,7 @@ public enum ScriptTypeEnum { ; private final int code; - private final String description; + private final String description; ScriptTypeEnum(int code, String description) { this.code = code; diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/SeaTunnelConnectorI18n.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/SeaTunnelConnectorI18n.java index 96e4780b9..aa3144bec 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/SeaTunnelConnectorI18n.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/SeaTunnelConnectorI18n.java @@ -30,13 +30,18 @@ public class SeaTunnelConnectorI18n { static { try { - CONNECTOR_I18N_CONFIG_EN = ConfigFactory.parseString(IOUtils.toString(SeaTunnelConnectorI18n.class.getResourceAsStream("/i18n_en.config"), StandardCharsets.UTF_8)); + CONNECTOR_I18N_CONFIG_EN = + ConfigFactory.parseString( + IOUtils.toString( + SeaTunnelConnectorI18n.class.getResourceAsStream( + "/i18n_en.config"), + StandardCharsets.UTF_8)); CONNECTOR_I18N_CONFIG_ZH = - ConfigFactory.parseString( - IOUtils.toString( - SeaTunnelConnectorI18n.class.getResourceAsStream( - "/i18n_zh.config"), - StandardCharsets.UTF_8)); + ConfigFactory.parseString( + IOUtils.toString( + SeaTunnelConnectorI18n.class.getResourceAsStream( + "/i18n_zh.config"), + StandardCharsets.UTF_8)); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserStatusEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserStatusEnum.java index c62b7e1cf..e8d533d17 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserStatusEnum.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserStatusEnum.java @@ -22,7 +22,7 @@ public enum UserStatusEnum { DISABLE(1, "disable"), ; private final int code; - private final String description; + private final String description; UserStatusEnum(int code, String description) { this.code = code; diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserTypeEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserTypeEnum.java index b583e6f48..6b8f3dfb2 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserTypeEnum.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserTypeEnum.java @@ -23,7 +23,7 @@ public enum UserTypeEnum { ; private final int code; - private final String description; + private final String description; UserTypeEnum(int code, String description) { this.code = code; diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/config/Swagger2.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/config/Swagger2.java index 3eb93a856..8e6df8543 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/config/Swagger2.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/config/Swagger2.java @@ -19,6 +19,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; + import springfox.documentation.builders.ApiInfoBuilder; import springfox.documentation.builders.PathSelectors; import springfox.documentation.builders.RequestHandlerSelectors; @@ -29,7 +30,7 @@ @Configuration @EnableSwagger2 -public class Swagger2{ +public class Swagger2 { @Bean public Docket createRestApi() { @@ -49,5 +50,4 @@ private ApiInfo apiInfo() { .termsOfServiceUrl("https://seatunnel.apache.org/") .build(); } - } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/AuthController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/AuthController.java index 39ab914ca..091190b04 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/AuthController.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/AuthController.java @@ -20,14 +20,15 @@ import org.apache.seatunnel.app.common.Result; import org.apache.seatunnel.app.service.IRoleService; -import io.swagger.annotations.ApiImplicitParam; -import io.swagger.annotations.ApiImplicitParams; -import io.swagger.annotations.ApiOperation; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; + import javax.annotation.Resource; import javax.validation.constraints.NotNull; @@ -35,8 +36,7 @@ @RestController public class AuthController { - @Resource - private IRoleService roleServiceImpl; + @Resource private IRoleService roleServiceImpl; @GetMapping("/userRole") @ApiOperation(value = "check relation between user and role", httpMethod = "GET") @@ -44,7 +44,9 @@ public class AuthController { @ApiImplicitParam(name = "username", value = "user name", dataType = "String"), @ApiImplicitParam(name = "roleName", value = "role name", dataType = "String"), }) - public Result userRole(@RequestParam("username") @NotNull String username, @RequestParam("roleName") @NotNull String roleName) { + public Result userRole( + @RequestParam("username") @NotNull String username, + @RequestParam("roleName") @NotNull String roleName) { final boolean b = roleServiceImpl.checkUserRole(username, roleName); return Result.success(b); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/ScriptController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/ScriptController.java index e1885e295..bf22b5a54 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/ScriptController.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/ScriptController.java @@ -31,8 +31,6 @@ import org.apache.seatunnel.app.domain.response.script.ScriptSimpleInfoRes; import org.apache.seatunnel.app.service.IScriptService; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PatchMapping; @@ -43,6 +41,9 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import springfox.documentation.annotations.ApiIgnore; import javax.annotation.Resource; @@ -53,22 +54,24 @@ @RequestMapping("/seatunnel/api/v1/script") @RestController public class ScriptController { - @Resource - private IScriptService iScriptService; + @Resource private IScriptService iScriptService; @PostMapping @ApiOperation(value = "add an script with content", httpMethod = "POST") - public Result createScript(@RequestBody @NotNull CreateScriptReq createScriptReq, - @ApiIgnore @UserId Integer operatorId) { + public Result createScript( + @RequestBody @NotNull CreateScriptReq createScriptReq, + @ApiIgnore @UserId Integer operatorId) { createScriptReq.setCreatorId(operatorId); return Result.success(iScriptService.createScript(createScriptReq)); } @PutMapping("/{scriptId}/content") @ApiOperation(value = "update script", httpMethod = "PUT") - public Result updateScriptContent(@ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") Integer scriptId, - @RequestBody @NotNull UpdateScriptContentReq req, - @ApiIgnore @UserId Integer operatorId) { + public Result updateScriptContent( + @ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") + Integer scriptId, + @RequestBody @NotNull UpdateScriptContentReq req, + @ApiIgnore @UserId Integer operatorId) { req.setScriptId(scriptId); req.setMenderId(operatorId); @@ -78,17 +81,20 @@ public Result updateScriptContent(@ApiParam(value = "script id", required @DeleteMapping("/{scriptId}") @ApiOperation(value = "delete script", httpMethod = "DELETE") - public Result delete(@ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") Integer scriptId) { + public Result delete( + @ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") + Integer scriptId) { iScriptService.delete(scriptId); return Result.success(); } @GetMapping @ApiOperation(value = "script list", httpMethod = "GET") - public Result> list(@ApiParam(value = "script name") @RequestParam(required = false) String name, - @ApiParam(value = "script status") @RequestParam(required = false) Byte status, - @ApiParam(value = "page num", required = true) @RequestParam Integer pageNo, - @ApiParam(value = "page size", required = true) @RequestParam Integer pageSize) { + public Result> list( + @ApiParam(value = "script name") @RequestParam(required = false) String name, + @ApiParam(value = "script status") @RequestParam(required = false) Byte status, + @ApiParam(value = "page num", required = true) @RequestParam Integer pageNo, + @ApiParam(value = "page size", required = true) @RequestParam Integer pageSize) { final ScriptListReq req = new ScriptListReq(); req.setName(name); @@ -100,19 +106,25 @@ public Result> list(@ApiParam(value = "script name @GetMapping("/{scriptId}/content") @ApiOperation(value = "fetch script content", httpMethod = "GET") - public Result fetchScriptContent(@ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") Integer scriptId) { + public Result fetchScriptContent( + @ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") + Integer scriptId) { return Result.success(iScriptService.fetchScriptContent(scriptId)); } @GetMapping("/{scriptId}") - public Result detail(@ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") Integer scriptId) { + public Result detail( + @ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") + Integer scriptId) { return Result.success(iScriptService.detail(scriptId)); } @PutMapping("/{scriptId}/param") @ApiOperation(value = "update script param", httpMethod = "PUT") - public Result updateScriptParam(@ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") Integer scriptId, - @RequestBody @NotNull UpdateScriptParamReq updateScriptParamReq) { + public Result updateScriptParam( + @ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") + Integer scriptId, + @RequestBody @NotNull UpdateScriptParamReq updateScriptParamReq) { updateScriptParamReq.setScriptId(scriptId); iScriptService.updateScriptParam(updateScriptParamReq); return Result.success(); @@ -120,14 +132,18 @@ public Result updateScriptParam(@ApiParam(value = "script id", required = @GetMapping("/{scriptId}/param") @ApiOperation(value = "fetch script param", httpMethod = "GET") - public Result> fetchScriptParam(@ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") Integer scriptId) { + public Result> fetchScriptParam( + @ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") + Integer scriptId) { return Result.success(iScriptService.fetchScriptParam(scriptId)); } @PatchMapping("/{scriptId}/publish") @ApiOperation(value = "publish script", httpMethod = "PATCH") - public Result publish(@ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") Integer scriptId, - @ApiIgnore @UserId Integer operatorId) { + public Result publish( + @ApiParam(value = "script id", required = true) @PathVariable(value = "scriptId") + Integer scriptId, + @ApiIgnore @UserId Integer operatorId) { final PublishScriptReq req = new PublishScriptReq(); req.setScriptId(scriptId); diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/TaskController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/TaskController.java index b66de54dc..7fc63526d 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/TaskController.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/TaskController.java @@ -29,8 +29,6 @@ import org.apache.seatunnel.app.domain.response.task.JobSimpleInfoRes; import org.apache.seatunnel.app.service.ITaskService; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PatchMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -39,6 +37,9 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import springfox.documentation.annotations.ApiIgnore; import javax.annotation.Resource; @@ -48,13 +49,13 @@ @RestController public class TaskController { - @Resource - private ITaskService iTaskService; + @Resource private ITaskService iTaskService; @PatchMapping("/{jobId}/recycle") @ApiOperation(value = "recycle job", httpMethod = "PATCH") - Result recycle(@ApiParam(value = "job id", required = true) @PathVariable(value = "jobId") Long jobId, - @ApiIgnore @UserId Integer operatorId) { + Result recycle( + @ApiParam(value = "job id", required = true) @PathVariable(value = "jobId") Long jobId, + @ApiIgnore @UserId Integer operatorId) { final RecycleScriptReq req = new RecycleScriptReq(); req.setJobId(jobId); req.setOperatorId(operatorId); @@ -65,9 +66,10 @@ Result recycle(@ApiParam(value = "job id", required = true) @PathVariable( @GetMapping("/job") @ApiOperation(value = "list job", httpMethod = "GET") - Result> listJob(@ApiParam(value = "job name") @RequestParam(required = false) String name, - @ApiParam(value = "page num", required = true) @RequestParam Integer pageNo, - @ApiParam(value = "page size", required = true) @RequestParam Integer pageSize) { + Result> listJob( + @ApiParam(value = "job name") @RequestParam(required = false) String name, + @ApiParam(value = "page num", required = true) @RequestParam Integer pageNo, + @ApiParam(value = "page size", required = true) @RequestParam Integer pageSize) { final JobListReq req = new JobListReq(); req.setName(name); req.setPageNo(pageNo); @@ -78,9 +80,11 @@ Result> listJob(@ApiParam(value = "job name") @Reques @GetMapping("/instance") @ApiOperation(value = "list instance", httpMethod = "GET") - Result> listInstance(@ApiParam(value = "job name", required = false) @RequestParam(required = false) String name, - @ApiParam(value = "page num", required = true) @RequestParam Integer pageNo, - @ApiParam(value = "page size", required = true) @RequestParam Integer pageSize) { + Result> listInstance( + @ApiParam(value = "job name", required = false) @RequestParam(required = false) + String name, + @ApiParam(value = "page num", required = true) @RequestParam Integer pageNo, + @ApiParam(value = "page size", required = true) @RequestParam Integer pageSize) { final InstanceListReq req = new InstanceListReq(); req.setName(name); req.setPageNo(pageNo); @@ -91,9 +95,11 @@ Result> listInstance(@ApiParam(value = "job name @PostMapping("/{objectId}/execute") @ApiOperation(value = "execute script temporary", httpMethod = "POST") - Result tmpExecute(@ApiParam(value = "object id", required = true) @PathVariable(value = "objectId") Long objectId, - @RequestBody @NotNull ExecuteReq req, - @ApiIgnore @UserId Integer operatorId) { + Result tmpExecute( + @ApiParam(value = "object id", required = true) @PathVariable(value = "objectId") + Long objectId, + @RequestBody @NotNull ExecuteReq req, + @ApiIgnore @UserId Integer operatorId) { req.setObjectId(objectId); req.setOperatorId(operatorId); @@ -102,13 +108,19 @@ Result tmpExecute(@ApiParam(value = "object id", required @GetMapping("/{taskInstanceId}") @ApiOperation(value = "query instance log", httpMethod = "GET") - Result queryInstanceLog(@ApiParam(value = "task instance id", required = true) @PathVariable(value = "taskInstanceId") Long taskInstanceId) { + Result queryInstanceLog( + @ApiParam(value = "task instance id", required = true) + @PathVariable(value = "taskInstanceId") + Long taskInstanceId) { return Result.success(iTaskService.queryInstanceLog(taskInstanceId)); } @PatchMapping("/{taskInstanceId}") @ApiOperation(value = "kill running instance", httpMethod = "POST") - Result kill(@ApiParam(value = "task instance id", required = true) @PathVariable(value = "taskInstanceId") Long taskInstanceId) { + Result kill( + @ApiParam(value = "task instance id", required = true) + @PathVariable(value = "taskInstanceId") + Long taskInstanceId) { iTaskService.kill(taskInstanceId); return Result.success(); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/UserController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/UserController.java index 6a818496a..106f23ff9 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/UserController.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/UserController.java @@ -27,8 +27,6 @@ import org.apache.seatunnel.app.domain.response.user.UserSimpleInfoRes; import org.apache.seatunnel.app.service.IUserService; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PatchMapping; @@ -40,6 +38,9 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; + import javax.annotation.Resource; import javax.validation.constraints.NotNull; @@ -47,8 +48,7 @@ @RestController public class UserController { - @Resource - private IUserService iUserService; + @Resource private IUserService iUserService; @PostMapping @ApiOperation(value = "add user", httpMethod = "POST") @@ -58,8 +58,10 @@ public Result add(@RequestBody @NotNull AddUserReq addReq) { @PutMapping("/{userId}") @ApiOperation(value = "update user", httpMethod = "PUT") - public Result update(@ApiParam(value = "user id", required = true) @PathVariable(value = "userId") Integer userId, - @RequestBody @NotNull UpdateUserReq updateReq) { + public Result update( + @ApiParam(value = "user id", required = true) @PathVariable(value = "userId") + Integer userId, + @RequestBody @NotNull UpdateUserReq updateReq) { updateReq.setUserId(userId); iUserService.update(updateReq); @@ -68,16 +70,19 @@ public Result update(@ApiParam(value = "user id", required = true) @PathVa @DeleteMapping("/{userId}") @ApiOperation(value = "delete user", httpMethod = "DELETE") - public Result delete(@ApiParam(value = "user id", required = true) @PathVariable(value = "userId") Integer userId) { + public Result delete( + @ApiParam(value = "user id", required = true) @PathVariable(value = "userId") + Integer userId) { iUserService.delete(userId); return Result.success(); } @GetMapping @ApiOperation(value = "user list", httpMethod = "GET") - public Result> list(@ApiParam(value = "user name") @RequestParam(required = false) String name, - @ApiParam(value = "page num", required = true) @RequestParam Integer pageNo, - @ApiParam(value = "page size", required = true) @RequestParam Integer pageSize) { + public Result> list( + @ApiParam(value = "user name") @RequestParam(required = false) String name, + @ApiParam(value = "page num", required = true) @RequestParam Integer pageNo, + @ApiParam(value = "page size", required = true) @RequestParam Integer pageSize) { final UserListReq req = new UserListReq(); req.setName(name); req.setPageNo(pageNo); @@ -88,14 +93,18 @@ public Result> list(@ApiParam(value = "user name") @ @PatchMapping("/{userId}/enable") @ApiOperation(value = "enable a user", httpMethod = "PATCH") - public Result enable(@ApiParam(value = "user id", required = true) @PathVariable(value = "userId") Integer userId) { + public Result enable( + @ApiParam(value = "user id", required = true) @PathVariable(value = "userId") + Integer userId) { iUserService.enable(userId); return Result.success(); } @PutMapping("/{userId}/disable") @ApiOperation(value = "disable a user", httpMethod = "PUT") - public Result disable(@ApiParam(value = "user id", required = true) @PathVariable(value = "userId") Integer userId) { + public Result disable( + @ApiParam(value = "user id", required = true) @PathVariable(value = "userId") + Integer userId) { iUserService.disable(userId); return Result.success(); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/RoleDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/RoleDaoImpl.java index ae0b32629..227f8f2b4 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/RoleDaoImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/RoleDaoImpl.java @@ -28,11 +28,10 @@ @Repository public class RoleDaoImpl implements IRoleDao { - @Resource - private RoleMapper roleMapper; + @Resource private RoleMapper roleMapper; @Override - public void add(Role role){ + public void add(Role role) { roleMapper.insert(role); } @@ -40,5 +39,4 @@ public void add(Role role){ public Role getByRoleName(String roleName) { return roleMapper.selectByRole(roleName); } - } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/RoleUserRelationDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/RoleUserRelationDaoImpl.java index 8bc66ddce..bee2cd7b3 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/RoleUserRelationDaoImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/RoleUserRelationDaoImpl.java @@ -28,17 +28,17 @@ @Repository public class RoleUserRelationDaoImpl implements IRoleUserRelationDao { - @Resource - private RoleUserRelationMapper roleUserRelationMapper; + @Resource private RoleUserRelationMapper roleUserRelationMapper; @Override - public void add(RoleUserRelation roleUserRelation){ + public void add(RoleUserRelation roleUserRelation) { roleUserRelationMapper.insert(roleUserRelation); } @Override public RoleUserRelation getByUserAndRole(Integer userId, Integer roleId) { - final RoleUserRelation roleUserRelation = roleUserRelationMapper.selectByUserIdAndRoleId(userId, roleId); + final RoleUserRelation roleUserRelation = + roleUserRelationMapper.selectByUserIdAndRoleId(userId, roleId); return roleUserRelation; } @@ -46,5 +46,4 @@ public RoleUserRelation getByUserAndRole(Integer userId, Integer roleId) { public void deleteByUserId(Integer userId) { roleUserRelationMapper.deleteByUserId(userId); } - } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/SchedulerConfigDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/SchedulerConfigDaoImpl.java index fa78042bd..1d6ac1fdd 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/SchedulerConfigDaoImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/SchedulerConfigDaoImpl.java @@ -29,8 +29,7 @@ @Repository public class SchedulerConfigDaoImpl implements ISchedulerConfigDao { - @Resource - private SchedulerConfigMapper schedulerConfigMapper; + @Resource private SchedulerConfigMapper schedulerConfigMapper; @Override public boolean exists(int scriptId) { diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/ScriptDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/ScriptDaoImpl.java index 7f18048d8..9e2ebdc5f 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/ScriptDaoImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/ScriptDaoImpl.java @@ -17,9 +17,6 @@ package org.apache.seatunnel.app.dal.dao.impl; -import static org.apache.seatunnel.server.common.SeatunnelErrorEnum.SCRIPT_ALREADY_EXIST; -import static com.google.common.base.Preconditions.checkState; - import org.apache.seatunnel.app.common.ScriptStatusEnum; import org.apache.seatunnel.app.dal.dao.IScriptDao; import org.apache.seatunnel.app.dal.entity.Script; @@ -37,16 +34,24 @@ import java.util.List; import java.util.Objects; +import static com.google.common.base.Preconditions.checkState; +import static org.apache.seatunnel.server.common.SeatunnelErrorEnum.SCRIPT_ALREADY_EXIST; + @Repository public class ScriptDaoImpl implements IScriptDao { - @Resource - private ScriptMapper scriptMapper; + @Resource private ScriptMapper scriptMapper; @Override public void checkScriptDuplicate(CheckScriptDuplicateDto dto) { - final Script script = scriptMapper.selectByNameAndCreatorAndStatusNotEq(dto.getName(), dto.getCreatorId(), (byte) ScriptStatusEnum.DELETED.getCode()); - checkState(Objects.isNull(script), String.format(SCRIPT_ALREADY_EXIST.getTemplate(), dto.getName())); + final Script script = + scriptMapper.selectByNameAndCreatorAndStatusNotEq( + dto.getName(), + dto.getCreatorId(), + (byte) ScriptStatusEnum.DELETED.getCode()); + checkState( + Objects.isNull(script), + String.format(SCRIPT_ALREADY_EXIST.getTemplate(), dto.getName())); } @Override @@ -69,7 +74,8 @@ public Script getScript(Integer id) { @Override public void updateScriptContent(UpdateScriptContentDto dto) { - scriptMapper.updateContentByPrimaryKey(dto.getId(), dto.getContent(), dto.getContentMd5(), dto.getMenderId()); + scriptMapper.updateContentByPrimaryKey( + dto.getId(), dto.getContent(), dto.getContentMd5(), dto.getMenderId()); } @Override @@ -82,7 +88,8 @@ public PageData