Skip to content

Commit

Permalink
Merge branch 'apache:dev' into new-connector-cassandra
Browse files Browse the repository at this point in the history
  • Loading branch information
FWLamb authored Nov 4, 2022
2 parents b3eb8e6 + 57c67af commit 825cb3b
Show file tree
Hide file tree
Showing 173 changed files with 3,719 additions and 1,407 deletions.
39 changes: 24 additions & 15 deletions .github/workflows/backend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,12 @@ jobs:
- "seatunnel-formats/**"
- "seatunnel-plugin-discovery/**"
- "seatunnel-transforms/**"
- "seatunnel-transforms-v2/**"
- "seatunnel-translation/**"
- "seatunnel-e2e/seatunnel-transforms-v2-e2e/**"
- "seatunnel-e2e/seatunnel-flink-e2e/**"
- "seatunnel-e2e/seatunnel-spark-e2e/**"
- "seatunnel-connectors/**"
- "plugin-mapping.properties"
- "pom.xml"
- "**/workflows/**"
- "**/tools/**"
Expand Down Expand Up @@ -199,23 +200,31 @@ jobs:
run: |
modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
modules=${modules: 1}
includes=`python tools/update_modules_check/update_modules_check.py tree $modules`
pl_modules=`python tools/update_modules_check/update_modules_check.py replace $modules`
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl $pl_modules > /tmp/sub_module.txt
sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
tree_modules="$modules$sub_modules"
includes=`python tools/update_modules_check/update_modules_check.py tree $tree_modules`
./mvnw -Pno_dist -D"e2e.dependency.skip"=false dependency:tree $includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt
modules=`python tools/update_modules_check/update_modules_check.py final /tmp/tree_out.txt`
echo $modules
echo "modules=$modules" >> $GITHUB_OUTPUT
build_modules=`python tools/update_modules_check/update_modules_check.py final_ut /tmp/tree_out.txt`
echo $build_modules
echo "modules=$build_modules" >> $GITHUB_OUTPUT
- name: Make integration test modules
id: it-modules
if: ${{ steps.filter.outputs.api == 'false' }}
run: |
modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.cv2-flink-e2e-modules.outputs.modules }}${{ steps.cv2-spark-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
modules=${modules: 1}
includes=`python tools/update_modules_check/update_modules_check.py tree $modules`
pl_modules=`python tools/update_modules_check/update_modules_check.py replace $modules`
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl $pl_modules > /tmp/sub_module.txt
sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
tree_modules="$modules$sub_modules"
includes=`python tools/update_modules_check/update_modules_check.py tree $tree_modules`
./mvnw -Pno_dist -D"e2e.dependency.skip"=false dependency:tree $includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt
modules=`python tools/update_modules_check/update_modules_check.py final /tmp/tree_out.txt`
echo $modules
echo "modules=$modules" >> $GITHUB_OUTPUT
build_modules=`python tools/update_modules_check/update_modules_check.py final_it /tmp/tree_out.txt`
echo $build_modules
echo "modules=$build_modules" >> $GITHUB_OUTPUT
dependency-license:
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
Expand All @@ -237,7 +246,6 @@ jobs:
./mvnw -B -q install -DskipTests
-D"maven.test.skip"=true
-D"maven.javadoc.skip"=true
-D"scalastyle.skip"=true
-D"checkstyle.skip"=true
-D"license.skipAddThirdParty"
- name: Check Dependencies Licenses
Expand All @@ -263,19 +271,20 @@ jobs:
- name: run all modules unit test
if: needs.changes.outputs.api == 'true'
run: |
./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx2048m

- name: run updated modules unit test
if: needs.changes.outputs.api == 'false' && needs.changes.outputs.ut-modules != ''
run: |
./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.ut-modules}} -am -amd -Pno_dist
./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.ut-modules}} -am -Pno_dist
env:
MAVEN_OPTS: -Xmx2048m

integration-test:
needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '')
runs-on: ${{ matrix.os }}
strategy:
matrix:
Expand All @@ -293,14 +302,14 @@ jobs:
- name: run all modules integration test
if: needs.changes.outputs.api == 'true'
run: |
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx2048m

- name: run updated modules integration test
if: needs.changes.outputs.api == 'false'
if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
run: |
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -amd -Pno_dist
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -Pno_dist
env:
MAVEN_OPTS: -Xmx2048m

2 changes: 1 addition & 1 deletion .github/workflows/codeql.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
timeout-minutes: 60
timeout-minutes: 120
env:
JAVA_TOOL_OPTIONS: -Xmx2G -Xms2G -Dhttp.keepAlive=false -Dmaven.test.skip=true -Dcheckstyle.skip=true -Dlicense.skipAddThirdParty=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120

Expand Down
7 changes: 1 addition & 6 deletions .github/workflows/schedule_backend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -143,9 +143,4 @@ jobs:
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx2048m
- name: run updated modules integration test
if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
run: |
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx2048m

78 changes: 78 additions & 0 deletions config/log4j2.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

property.file_path = ${sys:seatunnel.logs.path:-/tmp/seatunnel/logs}
property.file_name = ${sys:seatunnel.logs.file_name:-seatunnel}
property.file_split_size = 100MB
property.file_count = 100
property.file_ttl = 7d

rootLogger.level = INFO

############################ log output to console #############################
rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
############################ log output to console #############################
############################ log output to file #############################
#rootLogger.appenderRef.file.ref = fileAppender
############################ log output to file #############################

appender.consoleStdout.name = consoleStdoutAppender
appender.consoleStdout.type = CONSOLE
appender.consoleStdout.target = SYSTEM_OUT
appender.consoleStdout.layout.type = PatternLayout
appender.consoleStdout.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c - %m%n
appender.consoleStdout.filter.acceptLtWarn.type = ThresholdFilter
appender.consoleStdout.filter.acceptLtWarn.level = WARN
appender.consoleStdout.filter.acceptLtWarn.onMatch = DENY
appender.consoleStdout.filter.acceptLtWarn.onMismatch = ACCEPT

appender.consoleStderr.name = consoleStderrAppender
appender.consoleStderr.type = CONSOLE
appender.consoleStderr.target = SYSTEM_ERR
appender.consoleStderr.layout.type = PatternLayout
appender.consoleStderr.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c - %m%n
appender.consoleStderr.filter.acceptGteWarn.type = ThresholdFilter
appender.consoleStderr.filter.acceptGteWarn.level = WARN
appender.consoleStderr.filter.acceptGteWarn.onMatch = ACCEPT
appender.consoleStderr.filter.acceptGteWarn.onMismatch = DENY

appender.file.name = fileAppender
appender.file.type = RollingFile
appender.file.fileName = ${file_path}/${file_name}.log
appender.file.filePattern = ${file_path}/${file_name}.log.%d{yyyy-MM-dd}-%i
appender.file.append = true
appender.file.layout.type = PatternLayout
appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c - %m%n
appender.file.policies.type = Policies
appender.file.policies.time.type = TimeBasedTriggeringPolicy
appender.file.policies.time.modulate = true
appender.file.policies.size.type = SizeBasedTriggeringPolicy
appender.file.policies.size.size = ${file_split_size}
appender.file.strategy.type = DefaultRolloverStrategy
appender.file.strategy.fileIndex = nomax
appender.file.strategy.action.type = Delete
appender.file.strategy.action.basepath = ${file_path}
appender.file.strategy.action.maxDepth = 1
appender.file.strategy.action.condition.type = IfFileName
appender.file.strategy.action.condition.glob = ${file_name}.log*
appender.file.strategy.action.condition.nested_condition.type = IfAny
appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified
appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl}
appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount
appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}
6 changes: 6 additions & 0 deletions docs/en/connector-v2/sink/Clickhouse.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,3 +126,9 @@ sink {

### 2.3.0-beta 2022-10-20
- [Improve] Clickhouse Support Int128,Int256 Type ([3067](https://github.com/apache/incubator-seatunnel/pull/3067))

### next version

- [Improve] Clickhouse Sink support nest type and array type([3047](https://github.com/apache/incubator-seatunnel/pull/3047))

- [Improve] Clickhouse Sink support geo type([3141](https://github.com/apache/incubator-seatunnel/pull/3141))
5 changes: 5 additions & 0 deletions docs/en/connector-v2/source/Clickhouse.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,4 +93,9 @@ source {

- [Improve] Clickhouse Source random use host when config multi-host ([3108](https://github.com/apache/incubator-seatunnel/pull/3108))

### next version

- [Improve] Clickhouse Source support nest type and array type([3047](https://github.com/apache/incubator-seatunnel/pull/3047))

- [Improve] Clickhouse Source support geo type([3141](https://github.com/apache/incubator-seatunnel/pull/3141))

3 changes: 2 additions & 1 deletion docs/en/contribution/coding-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ This guide documents an overview of the current Apache SeaTunnel modules and bes
| seatunnel-engine | SeaTunnel engine module, seatunnel-engine is a new computational engine developed by the SeaTunnel Community that focuses on data synchronization. |
| seatunnel-formats | SeaTunnel formats module, used to offer the ability of formatting data |
| seatunnel-plugin-discovery | SeaTunnel plugin discovery module, used to offer the ability of loading SPI plugins from classpath |
| seatunnel-transforms | SeaTunnel transform plugin module |
| seatunnel-transforms | SeaTunnel transform V1 module, currently transform V1 is in a stable state, the community will continue to maintain it, but there will be no major feature updates |
| seatunnel-transforms-v2 | SeaTunnel transform V2 module, currently transform V2 is under development and the community will focus on it |
| seatunnel-translation | SeaTunnel translation module, used to adapt Connector V2 and other computing engines such as Spark Flink etc... |

## How to submit a high quality pull request
Expand Down
Loading

0 comments on commit 825cb3b

Please sign in to comment.