Skip to content

Commit

Permalink
Merge branch 'main' into 3731_update_doc
Browse files Browse the repository at this point in the history
  • Loading branch information
shaofengshi authored Jul 3, 2024
2 parents c46f6cf + 0463ea4 commit a006909
Show file tree
Hide file tree
Showing 1,602 changed files with 43,737 additions and 9,522 deletions.
55 changes: 55 additions & 0 deletions .asf.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

github:
description: World's most powerful open data catalog for building a high-performance, geo-distributed and federated metadata lake.
homepage: https://datastrato.ai/docs/
labels:
- metadata
- data-catalog
- datalake
- stratosphere
- federated-query
- lakehouse
- model-catalog
- metalake
- skycomputing
- ai-catalog
- opendatacatalog
features:
# Enable wiki for documentation
wiki: false
# Enable issues management
issues: true
# Enable projects for project management boards
projects: true
enabled_merge_buttons:
squash: true
merge: false
rebase: true
protected_branches:
main:
required_status_checks:
strict: true
required_pull_request_reviews:
dismiss_stale_reviews: true
required_approving_review_count: 1

notifications:
commits: [email protected]
issues: [email protected]
pullrequests: [email protected]
9 changes: 3 additions & 6 deletions .github/workflows/backend-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,11 @@ jobs:
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v3
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2
Expand All @@ -84,18 +85,14 @@ jobs:
run: |
./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
- name: Setup debug Github Action
if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }}
uses: csexton/debugger-action@master

- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Backend Integration Test
id: integrationTest
run: >
./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -P${{ matrix.backend }} -PskipWebITs
./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -P${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
-x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test
-x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test
-x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
Expand Down
34 changes: 12 additions & 22 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,11 @@ jobs:
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v3
- uses: actions/setup-java@v4
with:
java-version: 8
distribution: 'temurin'
cache: 'gradle'

- name: Build with Gradle
run: ./gradlew build -x test -PjdkVersion=8
Expand All @@ -81,39 +82,28 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v3

- uses: actions/setup-java@v3
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Test publish to local
run: ./gradlew publishToMavenLocal -x test -PjdkVersion=${{ matrix.java-version }}

- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Build with Gradle
run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }}
run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false -x :clients:client-python:build

- name: Upload unit tests report
uses: actions/upload-artifact@v3
if: failure()
with:
name: unit test report
path:
path: |
build/reports

- name: Jacoco Report to PR
id: jacoco
uses: madrapps/[email protected]
with:
paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml
token: ${{ secrets.GITHUB_TOKEN }}
min-coverage-overall: 40
min-coverage-changed-files: 60
title: 'Code Coverage Report'
debug-mode: false
update-comment: true
pass-emoji: ':green_circle:'
fail-emoji: ':red_circle:'
- name: Get the Coverage info
run: |
echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}"
echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }}"
catalogs/**/*.log
catalogs/**/*.tar
25 changes: 16 additions & 9 deletions .github/workflows/cron-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
needs: changes
if: needs.changes.outputs.source_changes == 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
timeout-minutes: 120
strategy:
matrix:
architecture: [linux/amd64]
Expand All @@ -62,31 +62,28 @@ jobs:
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v3
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'


- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Package Gravitino
run: |
./gradlew build -x test -PjdkVersion=${{ matrix.java-version }}
./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
- name: Setup debug Github Action
if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }}
uses: csexton/debugger-action@master

- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Integration Test
id: integrationTest
run: |
./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }}
./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
Expand All @@ -95,7 +92,17 @@ jobs:
name: integrate test reports
path: |
build/reports
integration-test/build/integration-test.log
integration-test/build/*.log
integration-test/build/*.tar
integration-test/build/trino-ci-container-log/hive/*.*
integration-test/build/trino-ci-container-log/hdfs/*.*
distribution/package/logs/gravitino-server.out
distribution/package/logs/gravitino-server.log
catalogs/**/*.log
catalogs/**/*.tar
distribution/**/*.log
spark-connector/v3.3/spark/build/spark-3.3-integration-test.log
spark-connector/v3.4/spark/build/spark-3.4-integration-test.log
spark-connector/v3.5/spark/build/spark-3.5-integration-test.log
flink-connector/build/flink-connector-integration-test.log
flink-connector/build/*.tar
10 changes: 9 additions & 1 deletion .github/workflows/docker-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,13 @@ on:
options:
- 'gravitino'
- 'gravitino-ci-hive'
- 'gravitino-ci-kerberos-hive'
- 'gravitino-ci-trino'
- 'gravitino-ci-doris'
- 'gravitino-ci-ranger'
- 'trino'
- 'hive'
- 'gravitino-ci-kerberos-hive'
- 'ranger'
tag:
description: 'Docker tag to apply to this image'
required: true
Expand Down Expand Up @@ -47,6 +49,9 @@ jobs:
elif [ "${{ github.event.inputs.image }}" == "gravitino-ci-doris" ]; then
echo "image_type=doris" >> $GITHUB_ENV
echo "image_name=datastrato/gravitino-ci-doris" >> $GITHUB_ENV
elif [ "${{ github.event.inputs.image }}" == "gravitino-ci-ranger" ]; then
echo "image_type=ranger" >> $GITHUB_ENV
echo "image_name=datastrato/gravitino-ci-ranger" >> $GITHUB_ENV
elif [ "${{ github.event.inputs.image }}" == "gravitino" ]; then
echo "image_type=gravitino" >> $GITHUB_ENV
echo "image_name=datastrato/gravitino" >> $GITHUB_ENV
Expand All @@ -56,6 +61,9 @@ jobs:
elif [ "${{ github.event.inputs.image }}" == "hive" ]; then
echo "image_type=hive" >> $GITHUB_ENV
echo "image_name=datastrato/hive" >> $GITHUB_ENV
elif [ "${{ github.event.inputs.image }}" == "ranger" ]; then
echo "image_type=ranger" >> $GITHUB_ENV
echo "image_name=datastrato/ranger" >> $GITHUB_ENV
fi
- name: Check publish Docker token
Expand Down
12 changes: 4 additions & 8 deletions .github/workflows/flink-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,11 @@ jobs:
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v3
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2
Expand All @@ -76,22 +77,17 @@ jobs:
- name: Package Gravitino
run: |
./gradlew build -x test -PjdkVersion=${{ matrix.java-version }}
./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
- name: Setup debug Github Action
if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }}
uses: csexton/debugger-action@master

- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Flink Integration Test
id: integrationTest
run: |
./gradlew --rerun-tasks -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
./gradlew --rerun-tasks -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
Expand Down
14 changes: 5 additions & 9 deletions .github/workflows/frontend-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ jobs:
needs: changes
if: needs.changes.outputs.source_changes == 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
timeout-minutes: 60
strategy:
matrix:
architecture: [linux/amd64]
Expand All @@ -65,10 +65,11 @@ jobs:
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v3
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2
Expand All @@ -79,22 +80,17 @@ jobs:
- name: Package Gravitino
run: |
./gradlew build -x test -PjdkVersion=${{ matrix.java-version }}
./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
- name: Setup debug Github Action
if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }}
uses: csexton/debugger-action@master

- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Frontend Integration Test
id: integrationTest
run: |
./gradlew --rerun-tasks -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
./gradlew --rerun-tasks -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
Expand Down
6 changes: 4 additions & 2 deletions .github/workflows/python-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,11 @@ jobs:
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v3
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2
Expand Down Expand Up @@ -80,4 +81,5 @@ jobs:
integration-test/build/integration-test.log
distribution/package/logs/gravitino-server.out
distribution/package/logs/gravitino-server.log
catalogs/**/*.log
catalogs/**/*.log
catalogs/**/*.tar
13 changes: 5 additions & 8 deletions .github/workflows/spark-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,11 @@ jobs:
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v3
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2
Expand All @@ -81,10 +82,6 @@ jobs:
run: |
./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
- name: Setup debug Github Action
if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }}
uses: csexton/debugger-action@master

- name: Free up disk space
run: |
dev/ci/util_free_space.sh
Expand All @@ -93,10 +90,10 @@ jobs:
id: integrationTest
run: |
if [ "${{ matrix.scala-version }}" == "2.12" ];then
./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.3:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
fi
./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.4:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.5:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
Expand Down
Loading

0 comments on commit a006909

Please sign in to comment.