diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties deleted file mode 100644 index 41d8213..0000000 --- a/.mvn/wrapper/maven-wrapper.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip -wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar diff --git a/docs/content.zh/docs/connectors/.keep b/docs/content.zh/docs/connectors/.keep new file mode 100644 index 0000000..e69de29 diff --git a/docs/content/docs/connectors/.keep b/docs/content/docs/connectors/.keep new file mode 100644 index 0000000..e69de29 diff --git a/flink-connector-clickhouse-e2e-test/pom.xml b/flink-connector-clickhouse-e2e-test/pom.xml new file mode 100644 index 0000000..ba5c30c --- /dev/null +++ b/flink-connector-clickhouse-e2e-test/pom.xml @@ -0,0 +1,24 @@ + + 4.0.0 + + org.apache.flink + flink-connector-clickhouse-parent + 1.17.1-SNAPSHOT + + + org.apache.flink + Flink : Connectors : Clickhouse : E2E Tests + flink-connector-clickhouse-e2e-test + https://flink.apache.org + jar + + + UTF-8 + + + + + + diff --git a/flink-connector-clickhouse/pom.xml b/flink-connector-clickhouse/pom.xml new file mode 100644 index 0000000..ac165f4 --- /dev/null +++ b/flink-connector-clickhouse/pom.xml @@ -0,0 +1,230 @@ + + 4.0.0 + + org.apache.flink + flink-connector-clickhouse-parent + 1.17.1-SNAPSHOT + + + flink-connector-clickhouse + Flink : Connectors : Clickhouse + + jar + + + + + + + org.apache.flink + flink-streaming-java + ${flink.version} + provided + + + + + + org.apache.flink + flink-connector-base + ${flink.version} + + + + + + + org.apache.flink + flink-table-api-java-bridge + ${flink.version} + provided + true + + + + + + ru.yandex.clickhouse + clickhouse-jdbc + ${clickhouse-jdbc.version} + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + + + + org.apache.flink + flink-core + ${flink.version} + test + test-jar + + + + org.apache.flink + flink-streaming-java + ${flink.version} + test + test-jar + + + + org.apache.flink + flink-test-utils + ${flink.version} + test + + + + org.apache.flink + flink-connector-test-utils + ${flink.version} + test + + + + org.apache.flink + flink-runtime + ${flink.version} + test-jar + test + + + + org.apache.flink + flink-connector-base + ${flink.version} + test + test-jar + + + + org.apache.flink + flink-metrics-jmx + ${flink.version} + test + + + + + org.apache.flink + flink-table-common + ${flink.version} + test-jar + test + + + + org.apache.flink + flink-table-test-utils + ${flink.version} + test + + + + org.apache.flink + flink-table-api-scala-bridge_${scala.binary.version} + ${flink.version} + test + + + org.apache.flink + flink-table-planner_${scala.binary.version} + ${flink.version} + test-jar + test + + + + + org.apache.flink + flink-json + ${flink.version} + test + + + org.apache.flink + flink-avro + ${flink.version} + test + + + org.apache.flink + flink-avro-confluent-registry + ${flink.version} + test + + + org.apache.flink + flink-csv + ${flink.version} + test + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + META-INF/LICENSE + META-INF/NOTICE + + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-test-sources + + test-jar-no-fork + + + + + false + + + META-INF/LICENSE + META-INF/NOTICE + + + + + + + + + diff --git a/src/main/java/org/apache/flink/connector/clickhouse/ClickHouseDynamicTableFactory.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/ClickHouseDynamicTableFactory.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/ClickHouseDynamicTableFactory.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/ClickHouseDynamicTableFactory.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/ClickHouseDynamicTableSink.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/ClickHouseDynamicTableSink.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/ClickHouseDynamicTableSink.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/ClickHouseDynamicTableSink.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/ClickHouseDynamicTableSource.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/ClickHouseDynamicTableSource.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/ClickHouseDynamicTableSource.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/ClickHouseDynamicTableSource.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/catalog/ClickHouseCatalog.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/catalog/ClickHouseCatalog.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/catalog/ClickHouseCatalog.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/catalog/ClickHouseCatalog.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/catalog/ClickHouseCatalogFactory.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/catalog/ClickHouseCatalogFactory.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/catalog/ClickHouseCatalogFactory.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/catalog/ClickHouseCatalogFactory.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/config/ClickHouseConfig.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/config/ClickHouseConfig.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/config/ClickHouseConfig.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/config/ClickHouseConfig.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/config/ClickHouseConfigOptions.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/config/ClickHouseConfigOptions.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/config/ClickHouseConfigOptions.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/config/ClickHouseConfigOptions.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/AbstractClickHouseInputFormat.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/AbstractClickHouseInputFormat.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/AbstractClickHouseInputFormat.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/AbstractClickHouseInputFormat.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/AbstractClickHouseOutputFormat.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/AbstractClickHouseOutputFormat.java similarity index 95% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/AbstractClickHouseOutputFormat.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/AbstractClickHouseOutputFormat.java index 6d0764d..1852c62 100644 --- a/src/main/java/org/apache/flink/connector/clickhouse/internal/AbstractClickHouseOutputFormat.java +++ b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/AbstractClickHouseOutputFormat.java @@ -123,8 +123,7 @@ protected void checkFlushException() { /** Builder for {@link ClickHouseBatchOutputFormat} and {@link ClickHouseShardOutputFormat}. */ public static class Builder { - private static final Logger LOG = - LoggerFactory.getLogger(AbstractClickHouseOutputFormat.Builder.class); + private static final Logger LOG = LoggerFactory.getLogger(Builder.class); private DataType[] fieldTypes; @@ -142,7 +141,7 @@ public static class Builder { public Builder() {} - public AbstractClickHouseOutputFormat.Builder withOptions(ClickHouseDmlOptions options) { + public Builder withOptions(ClickHouseDmlOptions options) { this.options = options; return this; } @@ -152,7 +151,7 @@ public Builder withConnectionProperties(Properties connectionProperties) { return this; } - public AbstractClickHouseOutputFormat.Builder withFieldTypes(DataType[] fieldTypes) { + public Builder withFieldTypes(DataType[] fieldTypes) { this.fieldTypes = fieldTypes; this.logicalTypes = Arrays.stream(fieldTypes) @@ -161,17 +160,17 @@ public AbstractClickHouseOutputFormat.Builder withFieldTypes(DataType[] fieldTyp return this; } - public AbstractClickHouseOutputFormat.Builder withFieldNames(String[] fieldNames) { + public Builder withFieldNames(String[] fieldNames) { this.fieldNames = fieldNames; return this; } - public AbstractClickHouseOutputFormat.Builder withPrimaryKey(String[] primaryKeys) { + public Builder withPrimaryKey(String[] primaryKeys) { this.primaryKeys = primaryKeys; return this; } - public AbstractClickHouseOutputFormat.Builder withPartitionKey(String[] partitionKeys) { + public Builder withPartitionKey(String[] partitionKeys) { this.partitionKeys = partitionKeys; return this; } diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseBatchInputFormat.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseBatchInputFormat.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseBatchInputFormat.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseBatchInputFormat.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseBatchOutputFormat.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseBatchOutputFormat.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseBatchOutputFormat.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseBatchOutputFormat.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseRowDataLookupFunction.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseRowDataLookupFunction.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseRowDataLookupFunction.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseRowDataLookupFunction.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseShardInputFormat.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseShardInputFormat.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseShardInputFormat.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseShardInputFormat.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseShardOutputFormat.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseShardOutputFormat.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseShardOutputFormat.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseShardOutputFormat.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseStatementFactory.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseStatementFactory.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseStatementFactory.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/ClickHouseStatementFactory.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/connection/ClickHouseConnectionProvider.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/connection/ClickHouseConnectionProvider.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/connection/ClickHouseConnectionProvider.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/connection/ClickHouseConnectionProvider.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/connection/ClickHouseStatementWrapper.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/connection/ClickHouseStatementWrapper.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/connection/ClickHouseStatementWrapper.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/connection/ClickHouseStatementWrapper.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/converter/ClickHouseConverterUtils.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/converter/ClickHouseConverterUtils.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/converter/ClickHouseConverterUtils.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/converter/ClickHouseConverterUtils.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/converter/ClickHouseRowConverter.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/converter/ClickHouseRowConverter.java similarity index 97% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/converter/ClickHouseRowConverter.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/converter/ClickHouseRowConverter.java index 2376f69..98910c7 100644 --- a/src/main/java/org/apache/flink/connector/clickhouse/internal/converter/ClickHouseRowConverter.java +++ b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/converter/ClickHouseRowConverter.java @@ -80,8 +80,7 @@ public void toExternal(RowData rowData, ClickHouseStatementWrapper statement) } } - protected ClickHouseRowConverter.DeserializationConverter createToInternalConverter( - LogicalType type) { + protected DeserializationConverter createToInternalConverter(LogicalType type) { switch (type.getTypeRoot()) { case NULL: return val -> null; @@ -134,8 +133,7 @@ protected ClickHouseRowConverter.DeserializationConverter createToInternalConver } } - protected ClickHouseRowConverter.SerializationConverter createToExternalConverter( - LogicalType type) { + protected SerializationConverter createToExternalConverter(LogicalType type) { switch (type.getTypeRoot()) { case BOOLEAN: return (val, index, statement) -> diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/executor/ClickHouseBatchExecutor.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/executor/ClickHouseBatchExecutor.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/executor/ClickHouseBatchExecutor.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/executor/ClickHouseBatchExecutor.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/executor/ClickHouseExecutor.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/executor/ClickHouseExecutor.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/executor/ClickHouseExecutor.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/executor/ClickHouseExecutor.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/executor/ClickHouseUpsertExecutor.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/executor/ClickHouseUpsertExecutor.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/executor/ClickHouseUpsertExecutor.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/executor/ClickHouseUpsertExecutor.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseConnectionOptions.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseConnectionOptions.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseConnectionOptions.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseConnectionOptions.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseDmlOptions.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseDmlOptions.java similarity index 80% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseDmlOptions.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseDmlOptions.java index a8f110f..3a47908 100644 --- a/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseDmlOptions.java +++ b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseDmlOptions.java @@ -123,78 +123,77 @@ public static class Builder { public Builder() {} - public ClickHouseDmlOptions.Builder withUrl(String url) { + public Builder withUrl(String url) { this.url = url; return this; } - public ClickHouseDmlOptions.Builder withUsername(String username) { + public Builder withUsername(String username) { this.username = username; return this; } - public ClickHouseDmlOptions.Builder withPassword(String password) { + public Builder withPassword(String password) { this.password = password; return this; } - public ClickHouseDmlOptions.Builder withDatabaseName(String databaseName) { + public Builder withDatabaseName(String databaseName) { this.databaseName = databaseName; return this; } - public ClickHouseDmlOptions.Builder withTableName(String tableName) { + public Builder withTableName(String tableName) { this.tableName = tableName; return this; } - public ClickHouseDmlOptions.Builder withBatchSize(int batchSize) { + public Builder withBatchSize(int batchSize) { this.batchSize = batchSize; return this; } - public ClickHouseDmlOptions.Builder withFlushInterval(Duration flushInterval) { + public Builder withFlushInterval(Duration flushInterval) { this.flushInterval = flushInterval; return this; } - public ClickHouseDmlOptions.Builder withMaxRetries(int maxRetries) { + public Builder withMaxRetries(int maxRetries) { this.maxRetries = maxRetries; return this; } - public ClickHouseDmlOptions.Builder withUpdateStrategy(SinkUpdateStrategy updateStrategy) { + public Builder withUpdateStrategy(SinkUpdateStrategy updateStrategy) { this.updateStrategy = updateStrategy; return this; } - public ClickHouseDmlOptions.Builder withUseLocal(Boolean useLocal) { + public Builder withUseLocal(Boolean useLocal) { this.useLocal = useLocal; return this; } - public ClickHouseDmlOptions.Builder withShardingStrategy( - SinkShardingStrategy shardingStrategy) { + public Builder withShardingStrategy(SinkShardingStrategy shardingStrategy) { this.shardingStrategy = shardingStrategy; return this; } - public ClickHouseDmlOptions.Builder withShardingKey(String shardingKey) { + public Builder withShardingKey(String shardingKey) { this.shardingKey = Collections.singletonList(shardingKey); return this; } - public ClickHouseDmlOptions.Builder withUseTableDef(boolean shardingUseTableDef) { + public Builder withUseTableDef(boolean shardingUseTableDef) { this.shardingUseTableDef = shardingUseTableDef; return this; } - public ClickHouseDmlOptions.Builder withIgnoreDelete(boolean ignoreDelete) { + public Builder withIgnoreDelete(boolean ignoreDelete) { this.ignoreDelete = ignoreDelete; return this; } - public ClickHouseDmlOptions.Builder withParallelism(Integer parallelism) { + public Builder withParallelism(Integer parallelism) { this.parallelism = parallelism; return this; } diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseReadOptions.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseReadOptions.java similarity index 84% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseReadOptions.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseReadOptions.java index a4fd40f..48d3a7f 100644 --- a/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseReadOptions.java +++ b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/options/ClickHouseReadOptions.java @@ -66,42 +66,42 @@ public static class Builder { private Long partitionLowerBound; private Long partitionUpperBound; - public ClickHouseReadOptions.Builder withUrl(String url) { + public Builder withUrl(String url) { this.url = url; return this; } - public ClickHouseReadOptions.Builder withUsername(String username) { + public Builder withUsername(String username) { this.username = username; return this; } - public ClickHouseReadOptions.Builder withPassword(String password) { + public Builder withPassword(String password) { this.password = password; return this; } - public ClickHouseReadOptions.Builder withDatabaseName(String databaseName) { + public Builder withDatabaseName(String databaseName) { this.databaseName = databaseName; return this; } - public ClickHouseReadOptions.Builder withTableName(String tableName) { + public Builder withTableName(String tableName) { this.tableName = tableName; return this; } - public ClickHouseReadOptions.Builder withUseLocal(boolean useLocal) { + public Builder withUseLocal(boolean useLocal) { this.useLocal = useLocal; return this; } - public ClickHouseReadOptions.Builder withPartitionColumn(String partitionColumn) { + public Builder withPartitionColumn(String partitionColumn) { this.partitionColumn = partitionColumn; return this; } - public ClickHouseReadOptions.Builder withPartitionNum(Integer partitionNum) { + public Builder withPartitionNum(Integer partitionNum) { this.partitionNum = partitionNum; return this; } diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/BalancedPartitioner.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/BalancedPartitioner.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/BalancedPartitioner.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/BalancedPartitioner.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/ClickHousePartitioner.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/ClickHousePartitioner.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/ClickHousePartitioner.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/ClickHousePartitioner.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/JavaHashPartitioner.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/JavaHashPartitioner.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/JavaHashPartitioner.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/JavaHashPartitioner.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/ShufflePartitioner.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/ShufflePartitioner.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/ShufflePartitioner.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/ShufflePartitioner.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/ValuePartitioner.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/ValuePartitioner.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/ValuePartitioner.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/partitioner/ValuePartitioner.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/ClusterSpec.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/ClusterSpec.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/schema/ClusterSpec.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/ClusterSpec.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/DistributedEngineFull.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/DistributedEngineFull.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/schema/DistributedEngineFull.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/DistributedEngineFull.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/Expression.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/Expression.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/schema/Expression.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/Expression.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/FieldExpr.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/FieldExpr.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/schema/FieldExpr.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/FieldExpr.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/FunctionExpr.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/FunctionExpr.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/schema/FunctionExpr.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/FunctionExpr.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/ReplicaSpec.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/ReplicaSpec.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/schema/ReplicaSpec.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/ReplicaSpec.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/ShardSpec.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/ShardSpec.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/internal/schema/ShardSpec.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/internal/schema/ShardSpec.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseBatchBetweenParametersProvider.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseBatchBetweenParametersProvider.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseBatchBetweenParametersProvider.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseBatchBetweenParametersProvider.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseBetweenParametersProvider.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseBetweenParametersProvider.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseBetweenParametersProvider.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseBetweenParametersProvider.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseNonParametersProvider.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseNonParametersProvider.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseNonParametersProvider.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseNonParametersProvider.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseParametersProvider.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseParametersProvider.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseParametersProvider.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseParametersProvider.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseShardBetweenParametersProvider.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseShardBetweenParametersProvider.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseShardBetweenParametersProvider.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseShardBetweenParametersProvider.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseShardTableParametersProvider.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseShardTableParametersProvider.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseShardTableParametersProvider.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/split/ClickHouseShardTableParametersProvider.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/util/ClickHouseJdbcUtil.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/util/ClickHouseJdbcUtil.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/util/ClickHouseJdbcUtil.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/util/ClickHouseJdbcUtil.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/util/ClickHouseUtil.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/util/ClickHouseUtil.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/util/ClickHouseUtil.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/util/ClickHouseUtil.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/util/DataTypeUtil.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/util/DataTypeUtil.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/util/DataTypeUtil.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/util/DataTypeUtil.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/util/FilterPushDownHelper.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/util/FilterPushDownHelper.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/util/FilterPushDownHelper.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/util/FilterPushDownHelper.java diff --git a/src/main/java/org/apache/flink/connector/clickhouse/util/SqlClause.java b/flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/util/SqlClause.java similarity index 100% rename from src/main/java/org/apache/flink/connector/clickhouse/util/SqlClause.java rename to flink-connector-clickhouse/src/main/java/org/apache/flink/connector/clickhouse/util/SqlClause.java diff --git a/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-connector-clickhouse/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory similarity index 100% rename from src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory rename to flink-connector-clickhouse/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory diff --git a/src/test/java/com/tiny/AppTest.java b/flink-connector-clickhouse/src/test/java/org/apache/flink/connector/clickhouse/AppTest.java similarity index 99% rename from src/test/java/com/tiny/AppTest.java rename to flink-connector-clickhouse/src/test/java/org/apache/flink/connector/clickhouse/AppTest.java index 732c15c..4a9c012 100644 --- a/src/test/java/com/tiny/AppTest.java +++ b/flink-connector-clickhouse/src/test/java/org/apache/flink/connector/clickhouse/AppTest.java @@ -1,4 +1,4 @@ -package com.tiny; +package org.apache.flink.connector.clickhouse; import org.apache.flink.connector.clickhouse.internal.partitioner.ValuePartitioner; import org.apache.flink.connector.clickhouse.internal.schema.ClusterSpec; diff --git a/flink-sql-connector-clickhouse/pom.xml b/flink-sql-connector-clickhouse/pom.xml new file mode 100644 index 0000000..75778c1 --- /dev/null +++ b/flink-sql-connector-clickhouse/pom.xml @@ -0,0 +1,126 @@ + + 4.0.0 + + org.apache.flink + flink-connector-clickhouse-parent + 1.17.1-SNAPSHOT + + + flink-sql-connector-clickhouse + Flink : Connectors : SQL : Clickhouse + https://flink.apache.org + + jar + + + UTF-8 + + + + + com.diffplug.spotless + spotless-maven-plugin + ${spotless-maven-plugin.version} + + + + 1.7 + + + + + + org.apache.flink,org.apache.flink.shaded,,javax,java,scala,\# + + + + + + + + spotless-check + validate + + check + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + package + + shade + + + false + + + org.apache.flink:flink-connector-base + org.apache.flink:flink-connector-clickhouse + + + + log4j:* + org.slf4j:* + com.google.code.findbugs:jsr305 + + + + + ru.yandex.clickhouse + ${shade.base}.ru.yandex.clickhouse + + + com.fasterxml + ${shade.base}.com.fasterxml + + + org.apache.commons + ${shade.base}.org.apache.commons + + + org.apache.http + ${shade.base}.org.apache.http + + + net.jpountz + ${shade.base}.net.jpountz + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + org.apache.maven.plugins + maven-release-plugin + ${maven-release-plugin.version} + + release-@{project.version} + true + + + + + diff --git a/mvnw b/mvnw deleted file mode 100755 index 07a7095..0000000 --- a/mvnw +++ /dev/null @@ -1,279 +0,0 @@ -#!/bin/sh -# ---------------------------------------------------------------------------- -# Licensed to Julian Hyde under one or more contributor license -# agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. -# -# Julian Hyde licenses this file to you under the Modified BSD License -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at: -# -# http://opensource.org/licenses/BSD-3-Clause -# ---------------------------------------------------------------------------- - -# ---------------------------------------------------------------------------- -# Maven2 Start Up Batch script -# -# Required ENV vars: -# ------------------ -# JAVA_HOME - location of a JDK home dir -# -# Optional ENV vars -# ----------------- -# M2_HOME - location of maven2's installed home dir -# MAVEN_OPTS - parameters passed to the Java VM when running Maven -# e.g. to debug Maven itself, use -# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -# MAVEN_SKIP_RC - flag to disable loading of mavenrc files -# ---------------------------------------------------------------------------- - -if [ -z "$MAVEN_SKIP_RC" ] ; then - - if [ -f /etc/mavenrc ] ; then - . /etc/mavenrc - fi - - if [ -f "$HOME/.mavenrc" ] ; then - . "$HOME/.mavenrc" - fi - -fi - -# OS specific support. $var _must_ be set to either true or false. -cygwin=false; -darwin=false; -mingw=false -case "`uname`" in - CYGWIN*) cygwin=true ;; - MINGW*) mingw=true;; - Darwin*) darwin=true - # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home - # See https://developer.apple.com/library/mac/qa/qa1170/_index.html - if [ -z "$JAVA_HOME" ]; then - if [ -x "/usr/libexec/java_home" ]; then - export JAVA_HOME="`/usr/libexec/java_home`" - else - export JAVA_HOME="/Library/Java/Home" - fi - fi - ;; -esac - -if [ -z "$JAVA_HOME" ] ; then - if [ -r /etc/gentoo-release ] ; then - JAVA_HOME=`java-config --jre-home` - fi -fi - -if [ -z "$M2_HOME" ] ; then - ## resolve links - $0 may be a link to maven's home - PRG="$0" - - # need this for relative symlinks - while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG="`dirname "$PRG"`/$link" - fi - done - - saveddir=`pwd` - - M2_HOME=`dirname "$PRG"`/.. - - # make it fully qualified - M2_HOME=`cd "$M2_HOME" && pwd` - - cd "$saveddir" - # echo Using m2 at $M2_HOME -fi - -# For Cygwin, ensure paths are in UNIX format before anything is touched -if $cygwin ; then - [ -n "$M2_HOME" ] && - M2_HOME=`cygpath --unix "$M2_HOME"` - [ -n "$JAVA_HOME" ] && - JAVA_HOME=`cygpath --unix "$JAVA_HOME"` - [ -n "$CLASSPATH" ] && - CLASSPATH=`cygpath --path --unix "$CLASSPATH"` -fi - -# For Mingw, ensure paths are in UNIX format before anything is touched -if $mingw ; then - [ -n "$M2_HOME" ] && - M2_HOME="`(cd "$M2_HOME"; pwd)`" - [ -n "$JAVA_HOME" ] && - JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" - # TODO classpath? -fi - -if [ -z "$JAVA_HOME" ]; then - javaExecutable="`which javac`" - if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=`which readlink` - if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then - if $darwin ; then - javaHome="`dirname \"$javaExecutable\"`" - javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" - else - javaExecutable="`readlink -f \"$javaExecutable\"`" - fi - javaHome="`dirname \"$javaExecutable\"`" - javaHome=`expr "$javaHome" : '\(.*\)/bin'` - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "$JAVACMD" ] ; then - if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - else - JAVACMD="`which java`" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." >&2 - echo " We cannot execute $JAVACMD" >&2 - exit 1 -fi - -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi - -CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher - -# traverses directory structure from process work directory to filesystem root -# first directory with .mvn subdirectory is considered project base directory -find_maven_basedir() { - - if [ -z "$1" ] - then - echo "Path not specified to find_maven_basedir" - return 1 - fi - - basedir="$1" - wdir="$1" - while [ "$wdir" != '/' ] ; do - if [ -d "$wdir"/.mvn ] ; then - basedir=$wdir - break - fi - # workaround for JBEAP-8937 (on Solaris 10/Sparc) - if [ -d "${wdir}" ]; then - wdir=`cd "$wdir/.."; pwd` - fi - # end of workaround - done - echo "${basedir}" -} - -# concatenates all lines of a file -concat_lines() { - if [ -f "$1" ]; then - echo "$(tr -s '\n' ' ' < "$1")" - fi -} - -BASE_DIR=`find_maven_basedir "$(pwd)"` -if [ -z "$BASE_DIR" ]; then - exit 1; -fi - -########################################################################################## -# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -# This allows using the maven wrapper in projects that prohibit checking in binary data. -########################################################################################## -if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then - if [ "$MVNW_VERBOSE" = true ]; then - echo "Found .mvn/wrapper/maven-wrapper.jar" - fi -else - if [ "$MVNW_VERBOSE" = true ]; then - echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." - fi - jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" - while IFS="=" read key value; do - case "$key" in (wrapperUrl) jarUrl="$value"; break ;; - esac - done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" - if [ "$MVNW_VERBOSE" = true ]; then - echo "Downloading from: $jarUrl" - fi - wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" - - if command -v wget > /dev/null; then - if [ "$MVNW_VERBOSE" = true ]; then - echo "Found wget ... using wget" - fi - wget "$jarUrl" -O "$wrapperJarPath" - elif command -v curl > /dev/null; then - if [ "$MVNW_VERBOSE" = true ]; then - echo "Found curl ... using curl" - fi - curl -o "$wrapperJarPath" "$jarUrl" - else - if [ "$MVNW_VERBOSE" = true ]; then - echo "Falling back to using Java to download" - fi - javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" - if [ -e "$javaClass" ]; then - if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then - if [ "$MVNW_VERBOSE" = true ]; then - echo " - Compiling MavenWrapperDownloader.java ..." - fi - # Compiling the Java class - ("$JAVA_HOME/bin/javac" "$javaClass") - fi - if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then - # Running the downloader - if [ "$MVNW_VERBOSE" = true ]; then - echo " - Running MavenWrapperDownloader.java ..." - fi - ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") - fi - fi - fi -fi -########################################################################################## -# End of extension -########################################################################################## - -export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} -if [ "$MVNW_VERBOSE" = true ]; then - echo $MAVEN_PROJECTBASEDIR -fi -MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" - -# For Cygwin, switch paths to Windows format before running java -if $cygwin; then - [ -n "$M2_HOME" ] && - M2_HOME=`cygpath --path --windows "$M2_HOME"` - [ -n "$JAVA_HOME" ] && - JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` - [ -n "$CLASSPATH" ] && - CLASSPATH=`cygpath --path --windows "$CLASSPATH"` - [ -n "$MAVEN_PROJECTBASEDIR" ] && - MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` -fi - -WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -exec "$JAVACMD" \ - $MAVEN_OPTS \ - -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ - "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ - ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/mvnw.cmd b/mvnw.cmd deleted file mode 100644 index a198777..0000000 --- a/mvnw.cmd +++ /dev/null @@ -1,154 +0,0 @@ -@REM ---------------------------------------------------------------------------- -@REM Licensed to Julian Hyde under one or more contributor license -@REM agreements. See the NOTICE file distributed with this work for -@REM additional information regarding copyright ownership. -@REM -@REM Julian Hyde licenses this file to you under the Modified BSD License -@REM (the "License"); you may not use this file except in compliance with -@REM the License. You may obtain a copy of the License at: -@REM -@REM http://opensource.org/licenses/BSD-3-Clause -@REM ---------------------------------------------------------------------------- - -@REM ---------------------------------------------------------------------------- -@REM Maven2 Start Up Batch script -@REM -@REM Required ENV vars: -@REM JAVA_HOME - location of a JDK home dir -@REM -@REM Optional ENV vars -@REM M2_HOME - location of maven2's installed home dir -@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands -@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending -@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven -@REM e.g. to debug Maven itself, use -@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files -@REM ---------------------------------------------------------------------------- - -@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' -@echo off -@REM set title of command window -title %0 -@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' -@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% - -@REM set %HOME% to equivalent of $HOME -if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") - -@REM Execute a user defined script before this one -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre -@REM check for pre script, once with legacy .bat ending and once with .cmd ending -if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" -if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" -:skipRcPre - -@setlocal - -set ERROR_CODE=0 - -@REM To isolate internal variables from possible post scripts, we use another setlocal -@setlocal - -@REM ==== START VALIDATION ==== -if not "%JAVA_HOME%" == "" goto OkJHome - -echo. -echo Error: JAVA_HOME not found in your environment. >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -:OkJHome -if exist "%JAVA_HOME%\bin\java.exe" goto init - -echo. -echo Error: JAVA_HOME is set to an invalid directory. >&2 -echo JAVA_HOME = "%JAVA_HOME%" >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -@REM ==== END VALIDATION ==== - -:init - -@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". -@REM Fallback to current working directory if not found. - -set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% -IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir - -set EXEC_DIR=%CD% -set WDIR=%EXEC_DIR% -:findBaseDir -IF EXIST "%WDIR%"\.mvn goto baseDirFound -cd .. -IF "%WDIR%"=="%CD%" goto baseDirNotFound -set WDIR=%CD% -goto findBaseDir - -:baseDirFound -set MAVEN_PROJECTBASEDIR=%WDIR% -cd "%EXEC_DIR%" -goto endDetectBaseDir - -:baseDirNotFound -set MAVEN_PROJECTBASEDIR=%EXEC_DIR% -cd "%EXEC_DIR%" - -:endDetectBaseDir - -IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig - -@setlocal EnableExtensions EnableDelayedExpansion -for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a -@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% - -:endReadAdditionalConfig - -SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" -set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" -set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" -FOR /F "tokens=1,2 delims==" %%A IN (%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties) DO ( - IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B -) - -@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -@REM This allows using the maven wrapper in projects that prohibit checking in binary data. -if exist %WRAPPER_JAR% ( - echo Found %WRAPPER_JAR% -) else ( - echo Couldn't find %WRAPPER_JAR%, downloading it ... - echo Downloading from: %DOWNLOAD_URL% - powershell -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; (New-Object Net.WebClient).DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')" - echo Finished downloading %WRAPPER_JAR% -) -@REM End of extension - -%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* -if ERRORLEVEL 1 goto error -goto end - -:error -set ERROR_CODE=1 - -:end -@endlocal & set ERROR_CODE=%ERROR_CODE% - -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost -@REM check for post script, once with legacy .bat ending and once with .cmd ending -if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" -if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" -:skipRcPost - -@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' -if "%MAVEN_BATCH_PAUSE%" == "on" pause - -if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% - -exit /B %ERROR_CODE% diff --git a/pom.xml b/pom.xml index 7b39881..5a45209 100644 --- a/pom.xml +++ b/pom.xml @@ -5,12 +5,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + + org.apache.flink + flink-connector-parent + 1.0.0 + + org.apache.flink - flink-connector-clickhouse - 1.16.0-SNAPSHOT + flink-connector-clickhouse-parent + 1.17.1-SNAPSHOT - flink-connector-clickhouse - http://www.example.com + Flink : Connectors : Clickhouse Parent + + flink-connector-clickhouse + flink-sql-connector-clickhouse + flink-connector-clickhouse-e2e-test + + pom + https://flink.apache.org + 2023 @@ -30,20 +43,18 @@ UTF-8 - 1.8 - 1.8 - - 8.14 + 11 + 11 4.13.2 0.3.1 2.12 - 1.16.0 + 1.17.1 provided - 30.1.1-jre + 32.1.3-jre 2.12.4 - 3.12.0 + 3.13.0 org.apache.flink.shaded.clickhouse - + flink-connector-clickhouse-parent 3.8.1 3.2.4 2.17 @@ -101,141 +112,66 @@ + + org.codehaus.mojo + exec-maven-plugin + false + + + org.apache.flink + flink-ci-tools + ${flink.version} + + + + org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${maven.compiler.source} - ${maven.compiler.target} - + maven-jar-plugin + + + + + io.github.zentol.japicmp + japicmp-maven-plugin + + + + org.apache.rat + apache-rat-plugin + false org.apache.maven.plugins maven-checkstyle-plugin - ${maven-checkstyle-plugin.version} - - - com.puppycrawl.tools - checkstyle - - ${checkstyle.version} - - - - - validate - validate - - check - - - - - /tools/maven/suppressions.xml - true - /tools/maven/checkstyle.xml - true - true - com.diffplug.spotless spotless-maven-plugin - ${spotless-maven-plugin.version} - - - - 1.7 - - - - - - org.apache.flink,org.apache.flink.shaded,,javax,java,scala,\# - - - - - - - - spotless-check - validate - - check - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - false - - - - log4j:* - org.slf4j:* - com.google.code.findbugs:jsr305 - - - - - ru.yandex.clickhouse - ${shade.base}.ru.yandex.clickhouse - - - com.fasterxml - ${shade.base}.com.fasterxml - - - org.apache.commons - ${shade.base}.org.apache.commons - - - org.apache.http - ${shade.base}.org.apache.http - - - net.jpountz - ${shade.base}.net.jpountz - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - + maven-compiler-plugin + + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.plugins + maven-enforcer-plugin + + org.apache.maven.plugins - maven-release-plugin - ${maven-release-plugin.version} - - release-@{project.version} - true - + maven-shade-plugin + + + + org.commonjava.maven.plugins + directory-maven-plugin diff --git a/tools/releasing/shared/.gitignore b/tools/releasing/shared/.gitignore new file mode 100644 index 0000000..29b636a --- /dev/null +++ b/tools/releasing/shared/.gitignore @@ -0,0 +1,2 @@ +.idea +*.iml \ No newline at end of file diff --git a/tools/releasing/shared/LICENSE b/tools/releasing/shared/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/tools/releasing/shared/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/releasing/shared/README.md b/tools/releasing/shared/README.md new file mode 100644 index 0000000..c1e3d0b --- /dev/null +++ b/tools/releasing/shared/README.md @@ -0,0 +1,91 @@ +This is a collection of release utils for [Apache Flink](https://flink.apache.org/) connectors. + +# Integration + +The scripts assume that they are integrated into a connector repo as a submodule into the connector repo +under `tools/releasing/shared`. + +# Usage + +Some scripts rely on environment variables to be set. +These are checked at the start of each script. +Any instance of `${some_variable}` in this document refers to an environment variable that is used by the respective +script. + +## check_environment.sh + +Runs some pre-release checks for the current environment, for example that all required programs are available. +This should be run once at the start of the release process. + +## release_snapshot_branch.sh + +Creates (and pushes!) a new snapshot branch for the current commit. +The branch name is automatically determined from the version in the pom. +This script should be called when work on a new major/minor version of the connector has started. + +## update_branch_version.sh + +Updates the version in the poms of the current branch to `${NEW_VERSION}`. + +## stage_source_release.sh + +Creates a source release from the current branch and pushes it via `svn` +to [dist.apache.org](https://dist.apache.org/repos/dist/dev/flink). +The version is automatically determined from the version in the pom. +The created `svn` directory will contain a `-rc${RC_NUM}` suffix. + +## stage_jars.sh + +Creates the jars from the current branch and deploys them to [repository.apache.org](https://repository.apache.org). +The version will be suffixed with the Flink minor version, extracted from`${FLINK_VERSION}`, to indicate the supported Flink version. +If a particular version of a connector supports multiple Flink versions then this script should be called multiple +times. + +## release_source_release.sh + +Copies the source release from the [SVN release directory](https://dist.apache.org/repops/dist/dev/flink) to the +[SVN release directory](https://dist.apache.org/repops/dist/release/flink) on [dist.apache.org](https://dist.apache.org). + +For safety purposes this script does not automatically determine the project and version from the current directory/branch/tag. + +``` +PROJECT=flink-connector-elasticsearch VERSION=3.0.0 RC_NUM=2 ./release_source_release.sh +``` + +## release_git_tag.sh + +Creates a release tag for the current branch and pushes it to GitHub. +The tag will be suffixed with `-rc${RC_NUM}`, if `${RC_NUM}` was set. +This script should only be used _after_ the `-SNAPSHOT` version suffix was removd via `update_branch_version.sh`. + +## update_japicmp_configuration.sh + +Sets the japicmp reference version in the pom of the current branch to `${NEW_VERSION}`, enables compatibility checks +for `@PublicEvolving` when used on snapshot branches an clears the list of exclusions. +This should be called after a release on the associated snapshot branch. If it was a minor release it should +additionally be called on the `main` branch. + +# Common workflow + +1. run `release_snapshot_branch.sh` +2. do some development work on the created snapshot branch +3. checkout a specific commit to create a release from +4. run `check_environment.sh` +5. run `update_branch_version.sh` +6. run `stage_source_release.sh` +7. run `stage_jars.sh` (once for each supported Flink version) +8. run `release_git_tag.sh` (with `RC_NUM`) +9. vote on release +10. finalize release or cancel and go back to step 2 +11. run `release_source_release.sh` +12. run `release_git_tag.sh` (without `RC_NUM`) +13. run `update_japicmp_configuration.sh` (on snapshot branch, and maybe `main`) + +# Script naming conventions + +| Prefix | Meaning | +|---------|------------------------------------------------------------------------| +| check | Verifies conditions without making any changes. | +| update | Applies modifications locally to the current branch. | +| stage | Publishes an artifact to an intermediate location for voting purposes. | +| release | Publishes an artifact to a user-facing location. | diff --git a/tools/releasing/shared/_init.sh b/tools/releasing/shared/_init.sh new file mode 100644 index 0000000..35b3111 --- /dev/null +++ b/tools/releasing/shared/_init.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# all scripts should contain this line + source ${SCRIPT_DIR}/_init.sh +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +set -o errexit +set -o nounset +set -o pipefail + +export SHELLOPTS + +########################### + +MVN=${MVN:-mvn} + +if [ "$(uname)" == "Darwin" ]; then + SHASUM="shasum -a 512" +else + SHASUM="sha512sum" +fi + +REMOTE=${REMOTE:-upstream} + +########################## + +SOURCE_DIR=$( cd -- "$( dirname -- "${SCRIPT_DIR}/../../../.." )" &> /dev/null && pwd ) +RELEASE_DIR=${SOURCE_DIR}/tools/releasing/release +ARTIFACTS_DIR=${SOURCE_DIR}/tools/releasing/release/artifacts + +SVN_DEV_DIR="https://dist.apache.org/repos/dist/dev/flink" +SVN_RELEASE_DIR="https://dist.apache.org/repos/dist/release/flink" \ No newline at end of file diff --git a/tools/releasing/shared/_utils.sh b/tools/releasing/shared/_utils.sh new file mode 100644 index 0000000..b63379a --- /dev/null +++ b/tools/releasing/shared/_utils.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +function check_variables_set { + any_missing=false + + for variable in "$@" + do + if [ -z "${!variable:-}" ]; then + echo "${variable} was not set." + any_missing=true + fi + done + + if [ ${any_missing} == true ]; then + exit 1 + fi +} + +function create_pristine_source { + source_dir=$1 + release_dir=$2 + + clone_dir="${release_dir}/tmp-clone" + clean_dir="${release_dir}/tmp-clean-clone" + rm -rf ${clone_dir} + rm -rf ${clean_dir} + # create a temporary git clone to ensure that we have a pristine source release + git clone "${source_dir}" "${clone_dir}" + + rsync -a \ + --exclude ".git" --exclude ".gitignore" --exclude ".gitattributes" --exclude ".gitmodules" --exclude ".github" \ + --exclude ".idea" --exclude "*.iml" \ + --exclude ".DS_Store" \ + --exclude ".asf.yaml" \ + --exclude "target" --exclude "tools/releasing/shared" \ + "${clone_dir}/" "${clean_dir}" + + rm -rf "${clone_dir}" + + echo "${clean_dir}" +} + +function get_pom_version { + echo $(${MVN} help:evaluate -Dexpression="project.version" -q -DforceStdout) +} + +function set_pom_version { + new_version=$1 + + ${MVN} org.codehaus.mojo:versions-maven-plugin:2.8.1:set -DnewVersion=${new_version} -DgenerateBackupPoms=false --quiet +} \ No newline at end of file diff --git a/tools/releasing/shared/check_environment.sh b/tools/releasing/shared/check_environment.sh new file mode 100755 index 0000000..b2ad38c --- /dev/null +++ b/tools/releasing/shared/check_environment.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +source "${SCRIPT_DIR}/_init.sh" + +EXIT_CODE=0 + +function check_program_available { + if program=$(command -v ${1}); then + printf "\t%-10s%s\n" "${1}" "using ${program}" + else + printf "\t%-10s%s\n" "${1}" "is not available." + EXIT_CODE=1 + fi +} + +echo "Checking program availability:" +check_program_available git +check_program_available tar +check_program_available rsync +check_program_available gpg +check_program_available perl +check_program_available sed +check_program_available svn +check_program_available ${MVN} +check_program_available ${SHASUM} + +function check_git_connectivity { + cd "${SOURCE_DIR}" + if git ls-remote --exit-code ${REMOTE} &> /dev/null; then + printf "\tUsing git remote '${REMOTE}'.\n" + else + printf "\tGit remote '${REMOTE}' is not available.\n" + printf "\tRun 'git remote add upstream https://github.com/apache/' or set a custom remote with the 'REMOTE' env variable.\n" + exit 1 + fi +} + +echo "Checking git remote availability:" +if ! (check_git_connectivity); then + EXIT_CODE=1 +fi + +if [ ${EXIT_CODE} == 0 ]; then + echo "All set! :)" +else + echo "At least one problem was found!" +fi +exit ${EXIT_CODE} diff --git a/tools/releasing/shared/release_git_tag.sh b/tools/releasing/shared/release_git_tag.sh new file mode 100755 index 0000000..6418282 --- /dev/null +++ b/tools/releasing/shared/release_git_tag.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +source ${SCRIPT_DIR}/_init.sh +source ${SCRIPT_DIR}/_utils.sh + +########################### + +RC_NUM=${RC_NUM:-none} + +########################### + +function create_release_tag { + cd "${SOURCE_DIR}" + + version=$(get_pom_version) + if [[ ${version} =~ -SNAPSHOT$ ]]; then + echo "Tags should not be created for SNAPSHOT versions. Use 'update_branch_version.sh' first." + exit 1 + fi + + tag=v${version} + if [ "$RC_NUM" != "none" ]; then + tag=${tag}-rc${RC_NUM} + fi + + git tag -s -m "v${tag}" ${tag} + + git push ${REMOTE} ${tag} +} + +(create_release_tag) diff --git a/tools/releasing/shared/release_snapshot_branch.sh b/tools/releasing/shared/release_snapshot_branch.sh new file mode 100755 index 0000000..5a4d274 --- /dev/null +++ b/tools/releasing/shared/release_snapshot_branch.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +source "${SCRIPT_DIR}/_init.sh" + +########################### + +function create_snapshot_branch { + cd "${SOURCE_DIR}" + + version=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout | sed "s/-SNAPSHOT//") + branch="v${version}" + + git checkout -b ${branch} + git push ${REMOTE} ${branch}:${branch} +} + +(create_snapshot_branch) diff --git a/tools/releasing/shared/release_source_release.sh b/tools/releasing/shared/release_source_release.sh new file mode 100755 index 0000000..4ee2342 --- /dev/null +++ b/tools/releasing/shared/release_source_release.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +source "${SCRIPT_DIR}/_init.sh" +source "${SCRIPT_DIR}/_utils.sh" + +########################### + +check_variables_set PROJECT VERSION RC_NUM + +########################### + +function release_source_release { + svn move -m "Release ${PROJECT} ${VERSION}" ${SVN_DEV_DIR}/${PROJECT}-${VERSION}-rc${RC_NUM} ${SVN_RELEASE_DIR}/${PROJECT}-${VERSION} +} + +(release_source_release) diff --git a/tools/releasing/shared/stage_jars.sh b/tools/releasing/shared/stage_jars.sh new file mode 100755 index 0000000..0dd180c --- /dev/null +++ b/tools/releasing/shared/stage_jars.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +source "${SCRIPT_DIR}/_init.sh" +source "${SCRIPT_DIR}/_utils.sh" + +########################### + +check_variables_set FLINK_VERSION + +########################### + +function deploy_staging_jars { + cd "${SOURCE_DIR}" + mkdir -p "${RELEASE_DIR}" + + project_version=$(get_pom_version) + if [[ ${project_version} =~ -SNAPSHOT$ ]]; then + echo "Jars should not be created for SNAPSHOT versions. Use 'update_branch_version.sh' first." + exit 1 + fi + flink_minor_version=$(echo ${FLINK_VERSION} | sed "s/.[0-9]\+$//") + version=${project_version}-${flink_minor_version} + + echo "Deploying jars v${version} to repository.apache.org" + echo "To revert this step, login to 'https://repository.apache.org' -> 'Staging repositories' -> Select repository -> 'Drop'" + + clone_dir=$(create_pristine_source "${SOURCE_DIR}" "${RELEASE_DIR}") + cd "${clone_dir}" + set_pom_version "${version}" + + options="-Prelease,docs-and-source -DskipTests -DretryFailedDeploymentCount=10" + ${MVN} clean deploy ${options} -Dflink.version=${FLINK_VERSION} + + cd "${RELEASE_DIR}" + rm -rf "${clone_dir}" +} + +(deploy_staging_jars) diff --git a/tools/releasing/shared/stage_source_release.sh b/tools/releasing/shared/stage_source_release.sh new file mode 100755 index 0000000..b8e659f --- /dev/null +++ b/tools/releasing/shared/stage_source_release.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +source "${SCRIPT_DIR}/_init.sh" +source "${SCRIPT_DIR}/_utils.sh" + +########################### + +check_variables_set RC_NUM + +########################### + +function create_source_release { + cd ${SOURCE_DIR} + mkdir -p ${RELEASE_DIR} + mkdir -p ${ARTIFACTS_DIR} + + project=${PWD##*/} + version=$(get_pom_version) + if [[ ${version} =~ -SNAPSHOT$ ]]; then + echo "Source releases should not be created for SNAPSHOT versions. Use 'update_branch_version.sh' first." + exit 1 + fi + + echo "Creating source release v${version}" + echo "To revert this step, run 'rm ${ARTIFACTS_DIR}'" + + clone_dir=$(create_pristine_source "${SOURCE_DIR}" "${RELEASE_DIR}") + versioned_dir="${ARTIFACTS_DIR}/${project}-${version}" + mv ${clone_dir} ${versioned_dir} + + cd "${ARTIFACTS_DIR}" + tar czf ${ARTIFACTS_DIR}/${project}-${version}-src.tgz ${versioned_dir##*/} + gpg --armor --detach-sig ${ARTIFACTS_DIR}/${project}-${version}-src.tgz + ${SHASUM} ${project}-${version}-src.tgz >${project}-${version}-src.tgz.sha512 + + rm -rf ${versioned_dir} +} + +function deploy_source_release { + cd ${SOURCE_DIR} + project=${PWD##*/} + version=$(get_pom_version)-rc${RC_NUM} + + release=${project}-${version} + + echo "Deploying source release v${version}" + echo "To revert this step, run 'svn delete ${SVN_DEV_DIR}/${release}'" + + svn_dir=${RELEASE_DIR}/svn + rm -rf ${svn_dir} + mkdir -p ${svn_dir} + cd ${svn_dir} + + svn checkout ${SVN_DEV_DIR} --depth=immediates + cd flink + mkdir ${release} + mv ${ARTIFACTS_DIR}/* ${release} + svn add ${release} + svn commit -m "Add ${release}" + + cd ${RELEASE_DIR} + rm -rf ${svn_dir} +} + +(create_source_release) +(deploy_source_release) diff --git a/tools/releasing/shared/update_branch_version.sh b/tools/releasing/shared/update_branch_version.sh new file mode 100755 index 0000000..0c43b57 --- /dev/null +++ b/tools/releasing/shared/update_branch_version.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +source "${SCRIPT_DIR}/_init.sh" +source "${SCRIPT_DIR}/_utils.sh" + +########################## + +check_variables_set NEW_VERSION + +########################### + +function update_branch_version { + cd "${SOURCE_DIR}" + + set_pom_version "${NEW_VERSION}" + + git commit -am "Update version to $NEW_VERSION" + + echo "Done." +} + +(update_branch_version) diff --git a/tools/releasing/shared/update_japicmp_configuration.sh b/tools/releasing/shared/update_japicmp_configuration.sh new file mode 100755 index 0000000..ce3784f --- /dev/null +++ b/tools/releasing/shared/update_japicmp_configuration.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source ${SCRIPT_DIR}/_init.sh +source ${SCRIPT_DIR}/_utils.sh + +########################### + +check_variables_set NEW_VERSION + +########################### + +# Idealized use-cases: +# Scenario A) New major release X.0.0 +# Premise: +# There is a main branch a version X.0-SNAPSHOT, with a japimp reference version of (X-1).Y.Z +# Release flow: +# - update the main to (X+1).0-SNAPSHOT, but keep the reference version intact since X.0.0 is not released (yet) +# - create X.0-SNAPSHOT branch, but keep the reference version intact since X.0.0 is not released (yet) +# - release X.0.0 +# - update the japicmp reference version of both main and X.0-SNAPSHOT to X.0.0 +# - enable stronger compatibility constraints for X.0-SNAPSHOT to ensure compatibility for PublicEvolving +# Scenario A) New minor release X.Y.0 +# Premise: +# There is a main branch with a version X.Y-SNAPSHOT, with a japicmp reference version of X.(Y-1).0 . +# Release flow: +# - update the main branch to X.(Y+1)-SNAPSHOT, but keep the reference version intact since X.Y.0 is not released (yet) +# - create X.Y-SNAPSHOT branch, but keep the reference version intact since X.Y.0 is not released (yet) +# - release X.Y.0 +# - update the japicmp reference version of both main and X.Y-SNAPSHOT to X.Y.0 +# - enable stronger compatibility constraints for X.Y-SNAPSHOT to ensure compatibility for PublicEvolving +# Scenario C) New patch release X.Y.Z +# Premise: +# There is a snapshot branch with a version X.Y-SNAPSHOT, with a japicmp reference version of X.Y.(Z-1) +# Release flow: +# - release X.Y.Z +# - update the japicmp reference version of X.Y-SNAPSHOT to X.Y.Z + +function enable_public_evolving_compatibility_checks() { + perl -pi -e 's##${1}#' pom.xml + perl -pi -e 's#\t+\@org.apache.flink.annotation.PublicEvolving.*\n##' pom.xml +} + +function set_japicmp_reference_version() { + local version=$1 + + perl -pi -e 's#().*()#${1}'${version}'${2}#' pom.xml +} + +function clear_exclusions() { + exclusion_start=$(($(sed -n '/