diff --git a/.asf.yaml b/.asf.yaml new file mode 100644 index 00000000000..4dd25e0c0fa --- /dev/null +++ b/.asf.yaml @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +github: + description: World's most powerful open data catalog for building a high-performance, geo-distributed and federated metadata lake. + homepage: https://datastrato.ai/docs/ + labels: + - metadata + - data-catalog + - datalake + - stratosphere + - federated-query + - lakehouse + - model-catalog + - metalake + - skycomputing + - ai-catalog + - opendatacatalog + features: + # Enable wiki for documentation + wiki: false + # Enable issues management + issues: true + # Enable projects for project management boards + projects: true + enabled_merge_buttons: + squash: true + merge: false + rebase: true + protected_branches: + main: + required_status_checks: + strict: true + required_pull_request_reviews: + dismiss_stale_reviews: true + required_approving_review_count: 1 + +notifications: + commits: commits@gravitino.apache.org + issues: commits@gravitino.apache.org + pullrequests: commits@gravitino.apache.org diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index be5c0a620b0..da534f467db 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -67,10 +67,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: ${{ matrix.java-version }} distribution: 'temurin' + cache: 'gradle' - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -84,10 +85,6 @@ jobs: run: | ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - name: Setup debug Github Action - if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }} - uses: csexton/debugger-action@master - - name: Free up disk space run: | dev/ci/util_free_space.sh @@ -95,7 +92,7 @@ jobs: - name: Backend Integration Test id: integrationTest run: > - ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -P${{ matrix.backend }} -PskipWebITs + ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -P${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 32ca9a1adc7..cbb9eaffb68 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,10 +59,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: 8 distribution: 'temurin' + cache: 'gradle' - name: Build with Gradle run: ./gradlew build -x test -PjdkVersion=8 @@ -81,39 +82,28 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: ${{ matrix.java-version }} distribution: 'temurin' + cache: 'gradle' - name: Test publish to local run: ./gradlew publishToMavenLocal -x test -PjdkVersion=${{ matrix.java-version }} + - name: Free up disk space + run: | + dev/ci/util_free_space.sh + - name: Build with Gradle - run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} + run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false -x :clients:client-python:build - name: Upload unit tests report uses: actions/upload-artifact@v3 if: failure() with: name: unit test report - path: + path: | build/reports - - - name: Jacoco Report to PR - id: jacoco - uses: madrapps/jacoco-report@v1.6.1 - with: - paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml - token: ${{ secrets.GITHUB_TOKEN }} - min-coverage-overall: 40 - min-coverage-changed-files: 60 - title: 'Code Coverage Report' - debug-mode: false - update-comment: true - pass-emoji: ':green_circle:' - fail-emoji: ':red_circle:' - - name: Get the Coverage info - run: | - echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }}" + catalogs/**/*.log + catalogs/**/*.tar diff --git a/.github/workflows/cron-integration-test.yml b/.github/workflows/cron-integration-test.yml index a0f97a0abd3..195e1b6e97b 100644 --- a/.github/workflows/cron-integration-test.yml +++ b/.github/workflows/cron-integration-test.yml @@ -50,7 +50,7 @@ jobs: needs: changes if: needs.changes.outputs.source_changes == 'true' runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 strategy: matrix: architecture: [linux/amd64] @@ -62,23 +62,20 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: ${{ matrix.java-version }} distribution: 'temurin' + cache: 'gradle' + - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Package Gravitino run: | - ./gradlew build -x test -PjdkVersion=${{ matrix.java-version }} ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - name: Setup debug Github Action - if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }} - uses: csexton/debugger-action@master - - name: Free up disk space run: | dev/ci/util_free_space.sh @@ -86,7 +83,7 @@ jobs: - name: Integration Test id: integrationTest run: | - ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} + ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false - name: Upload integrate tests reports uses: actions/upload-artifact@v3 @@ -95,7 +92,17 @@ jobs: name: integrate test reports path: | build/reports - integration-test/build/integration-test.log + integration-test/build/*.log + integration-test/build/*.tar + integration-test/build/trino-ci-container-log/hive/*.* + integration-test/build/trino-ci-container-log/hdfs/*.* distribution/package/logs/gravitino-server.out distribution/package/logs/gravitino-server.log catalogs/**/*.log + catalogs/**/*.tar + distribution/**/*.log + spark-connector/v3.3/spark/build/spark-3.3-integration-test.log + spark-connector/v3.4/spark/build/spark-3.4-integration-test.log + spark-connector/v3.5/spark/build/spark-3.5-integration-test.log + flink-connector/build/flink-connector-integration-test.log + flink-connector/build/*.tar diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 22e9b5c7325..243a3478a93 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -11,11 +11,13 @@ on: options: - 'gravitino' - 'gravitino-ci-hive' + - 'gravitino-ci-kerberos-hive' - 'gravitino-ci-trino' - 'gravitino-ci-doris' + - 'gravitino-ci-ranger' - 'trino' - 'hive' - - 'gravitino-ci-kerberos-hive' + - 'ranger' tag: description: 'Docker tag to apply to this image' required: true @@ -47,6 +49,9 @@ jobs: elif [ "${{ github.event.inputs.image }}" == "gravitino-ci-doris" ]; then echo "image_type=doris" >> $GITHUB_ENV echo "image_name=datastrato/gravitino-ci-doris" >> $GITHUB_ENV + elif [ "${{ github.event.inputs.image }}" == "gravitino-ci-ranger" ]; then + echo "image_type=ranger" >> $GITHUB_ENV + echo "image_name=datastrato/gravitino-ci-ranger" >> $GITHUB_ENV elif [ "${{ github.event.inputs.image }}" == "gravitino" ]; then echo "image_type=gravitino" >> $GITHUB_ENV echo "image_name=datastrato/gravitino" >> $GITHUB_ENV @@ -56,6 +61,9 @@ jobs: elif [ "${{ github.event.inputs.image }}" == "hive" ]; then echo "image_type=hive" >> $GITHUB_ENV echo "image_name=datastrato/hive" >> $GITHUB_ENV + elif [ "${{ github.event.inputs.image }}" == "ranger" ]; then + echo "image_type=ranger" >> $GITHUB_ENV + echo "image_name=datastrato/ranger" >> $GITHUB_ENV fi - name: Check publish Docker token diff --git a/.github/workflows/flink-integration-test.yml b/.github/workflows/flink-integration-test.yml index 818618eebbc..ba7648c7214 100644 --- a/.github/workflows/flink-integration-test.yml +++ b/.github/workflows/flink-integration-test.yml @@ -62,10 +62,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: ${{ matrix.java-version }} distribution: 'temurin' + cache: 'gradle' - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -76,13 +77,8 @@ jobs: - name: Package Gravitino run: | - ./gradlew build -x test -PjdkVersion=${{ matrix.java-version }} ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - name: Setup debug Github Action - if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }} - uses: csexton/debugger-action@master - - name: Free up disk space run: | dev/ci/util_free_space.sh @@ -90,8 +86,8 @@ jobs: - name: Flink Integration Test id: integrationTest run: | - ./gradlew --rerun-tasks -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**" - ./gradlew --rerun-tasks -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**" + ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**" + ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**" - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml index dfd192b36cd..7b0315e4677 100644 --- a/.github/workflows/frontend-integration-test.yml +++ b/.github/workflows/frontend-integration-test.yml @@ -55,7 +55,7 @@ jobs: needs: changes if: needs.changes.outputs.source_changes == 'true' runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: matrix: architecture: [linux/amd64] @@ -65,10 +65,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: ${{ matrix.java-version }} distribution: 'temurin' + cache: 'gradle' - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -79,13 +80,8 @@ jobs: - name: Package Gravitino run: | - ./gradlew build -x test -PjdkVersion=${{ matrix.java-version }} ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - name: Setup debug Github Action - if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }} - uses: csexton/debugger-action@master - - name: Free up disk space run: | dev/ci/util_free_space.sh @@ -93,8 +89,8 @@ jobs: - name: Frontend Integration Test id: integrationTest run: | - ./gradlew --rerun-tasks -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**" - ./gradlew --rerun-tasks -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**" + ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**" + ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**" - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml index 3840bfc3f0d..f2e5fd4edd4 100644 --- a/.github/workflows/python-integration-test.yml +++ b/.github/workflows/python-integration-test.yml @@ -49,10 +49,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: ${{ matrix.java-version }} distribution: 'temurin' + cache: 'gradle' - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -80,4 +81,5 @@ jobs: integration-test/build/integration-test.log distribution/package/logs/gravitino-server.out distribution/package/logs/gravitino-server.log - catalogs/**/*.log \ No newline at end of file + catalogs/**/*.log + catalogs/**/*.tar \ No newline at end of file diff --git a/.github/workflows/spark-integration-test.yml b/.github/workflows/spark-integration-test.yml index 39bf66c5876..06dd385af47 100644 --- a/.github/workflows/spark-integration-test.yml +++ b/.github/workflows/spark-integration-test.yml @@ -64,10 +64,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: ${{ matrix.java-version }} distribution: 'temurin' + cache: 'gradle' - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -81,10 +82,6 @@ jobs: run: | ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - name: Setup debug Github Action - if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }} - uses: csexton/debugger-action@master - - name: Free up disk space run: | dev/ci/util_free_space.sh @@ -93,10 +90,10 @@ jobs: id: integrationTest run: | if [ "${{ matrix.scala-version }}" == "2.12" ];then - ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.3:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**" + ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**" fi - ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.4:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**" - ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.5:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**" + ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**" + ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**" - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 9c0efc368f6..c83f1d12e38 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,6 +1,20 @@ # Contributor Code of Conduct diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a2df0ee61a3..be61fb11846 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,20 @@ # Contributing to Gravitino @@ -13,6 +27,14 @@ Before you get started, please read and follow these guidelines to ensure a smoo - [Getting Started](#getting-started) - [Fork the Repository](#fork-the-repository) + - [Development Setup](#development-setup) + - [Using IntelliJ (Optional)](#using-intellij-optional) + - [Using VS Code (Optional)](#using-vs-code-optional) + - [Handling Memory Issues in WSL](#handling-memory-issues-in-wsl) +- [Project Overview and Policies](#project-overview-and-policies) + - [Project Overview](#project-overview) + - [Management Policies](#management-policies) + - [Future Development Directions](#future-development-directions) - [Contribution Guidelines](#contribution-guidelines) - [Code of Conduct](#code-of-conduct) - [Reporting Bugs](#reporting-bugs) @@ -20,7 +42,7 @@ Before you get started, please read and follow these guidelines to ensure a smoo - [Good First Issues](#good-first-issues) - [Working on Issues](#working-on-issues) - [Creating Pull Requests](#creating-pull-requests) -- [Development Setup](#development-setup) + - [The Review Process](#the-review-process) - [Testing](#testing) - [Coding Standards](#coding-standards) - [Community and Communication](#community-and-communication) @@ -37,33 +59,181 @@ git clone https://github.com/datastrato/gravitino.git cd gravitino ``` -Now you are ready to start making contributions. +### Development Setup + +Once you have cloned the [GitHub repository](https://github.com/datastrato/gravitino), see [how to build](/docs/how-to-build.md) for instructions on how to build, or you can use the provided docker images at [Datastrato's DockerHub repository](https://hub.docker.com/u/datastrato). + +To stop and start a local Gravitino server via `bin/gravitino.sh start` and `bin/gravitino.sh stop` in a Gravitino distribution, see [how to build](/docs/how-to-build.md) for more instructions. + +### Using IntelliJ (Optional) + +**On Windows:** + +1. Open the Gravitino project that was just downloaded. + +2. Go to `File > Project Structure > Project` and change to the SDK you downloaded in WSL. + +3. If the SDK does not appear, manually add the SDK. + +4. To find the SDK location, run this command in WSL: + + **On Ubuntu (WSL):** + ```sh + which java + ``` + +IntelliJ IDEA is an integrated development environment (IDE) for Java development. Setting the project SDK ensures that IntelliJ uses the correct Java version for building and running the project. + +You can open up WSL in the IntelliJ terminal. Find the down arrow and select ubuntu. + +### Using VS Code (Optional) + +#### Set up WSL Extension in VSCode + +**On Windows:** + +1. Open VSCode extension marketplace, search for and install **WSL**. + +2. On Windows, press `Ctrl+Shift+P` to open the command palette, and run `Shell Command: Install 'code' command in PATH`. + +Installing the WSL extension in VSCode allows you to open and edit files in your WSL environment directly from VSCode. Adding the `code` command to your PATH enables you to open VSCode from the WSL terminal. + +#### Verify and Configure Environment Variables + +**On Windows:** + +1. Add VSCode path to the environment variables. The default installation path for VSCode is usually: + + ```plaintext + C:\Users\\AppData\Local\Programs\Microsoft VS Code\bin + ``` + + Replace `` with your actual Windows username. + + Example: + + ```plaintext + C:\Users\epic\AppData\Local\Programs\Microsoft VS Code\bin + ``` + +Adding VSCode to the environment variables ensures that you can open VSCode from any command prompt or terminal window. + +**On Ubuntu (WSL):** + +```sh +code --version +cd gravitino +code . +``` + +Running `code --version` verifies that the `code` command is available. Using `code .` opens the current directory in VSCode. + +#### Open a WSL Project in Windows VSCode + +**On Ubuntu (WSL):** + +1. **Navigate to Your Project Directory** + + Use the terminal to navigate to the directory of your project. For example: + + ```sh + cd gravitino + ``` + +2. **Open the Project in VSCode** + + In the WSL terminal, type the following command to open the current directory in VSCode: + + ```sh + code . + ``` + + This command will open the current WSL directory in VSCode on Windows. If you haven't added `code` to your path, follow these steps: + +- Open VSCode on Windows. + +- Press `Ctrl+Shift+P` to open the command palette. + +- Type and select `Shell Command: Install 'code' command in PATH`. + +3. **Ensure Remote - WSL is Active** + + When VSCode opens, you should see a green bottom-left corner indicating that VSCode is connected to WSL. If it isn't, click on the green area and select `Remote-WSL: New Window` or `Remote-WSL: Reopen Folder in WSL`. + +4. **Edit and Develop Your Project** + + You can now edit and develop your project files in VSCode as if they were local files. The Remote - WSL extension seamlessly bridges the file system between Windows and WSL. + +### Handling Memory Issues in WSL + +If you ran into a memory issue when using WSL, here are some solutions to resolve it. + +1. **Shut down WSL** + If your WSL is open, you can shut it down in Windows PowerShell using the following command: + + ```powershell + wsl --shutdown + ``` + +2. **Navigate to user folder** + + Open up File Explorer and navigate to `C:\Users\`. + +3. **Create the `.wslconfig` file** + + Open up Notepad or another text editor and input the following: + + ```plaintext + [wsl2] + memory=4GB # Limits VM memory in WSL 2 up to 4GB + processors=4 # Makes the WSL 2 VM use four virtual processors + ``` + + *The memory and processor usage can be changed depending on your system's hardware.* + +4. **Save the file** + + Save the file as `".wslconfig"`. Be sure to include the quotes to let Windows know that this isn't a text file. + +## Project Overview and Policies + +### Project Overview + +For an overview of the project, see [README.md](README.md). + +### Management Policies + +For project management policies, refer to [GOVERNANCE.md](GOVERNANCE.md). + +### Future Development Directions + +For future development directions, refer to the [ROADMAP.md](ROADMAP.md) document. ## Contribution guidelines -### Code of conduct +### Code of Conduct Please read and follow the [Code of Conduct](CODE_OF_CONDUCT.md). Gravitino provides a welcoming and inclusive environment for all contributors. ### Reporting bugs -If you find a bug in Gravitino, please open an issue on GitHub. Be sure to include as much detail as possible, such as a clear description, steps to reproduce, and your environment. Please follow the template provided. +If you find a bug in Gravitino, please open an issue on GitHub. Be sure to include as much detail as possible, such as a clear description, steps to reproduce, and your environment. Please follow the template provided. If you encounter a security issue, please refer to [SECURITY.md](SECURITY.md). ### Suggesting enhancements If you have ideas for enhancements or new features, feel free to create an issue to discuss them. Gravitino welcomes suggestions and provides prompt feedback on their feasibility and relevance. -### Good first issues +### Good First Issues -If you are new to open source or can't find something to work on check out the [Good First Issues list](https://github.com/datastrato/gravitino/contribute). +If you are new to open source or can't find something to work on, check out the [Good First Issues list](https://github.com/datastrato/gravitino/contribute). -### Working on issues +### Working on Issues Check out the list of open issues and find one that interests you. You can also comment on an issue to indicate that you're working on it. Please keep the issue updated with your progress. -## Creating pull requests +## Creating Pull Requests -Create a new branch from ``main`` for your changes: +Create a new branch from `main` for your changes: ```bash git checkout -b your-branch-name @@ -81,13 +251,11 @@ Push your changes to your fork on GitHub: git push your-branch-name ``` -After you have pushed your changes, create a pull request (PR) in the Gravitino repository. Be sure to provide a detailed description of your changes, reference any related issues and please follow the template provided. Gravitino's maintainers evaluate pull requests, offer feedback, and assist in merging project changes. +After you have pushed your changes, create a pull request (PR) in the Gravitino repository. Be sure to provide a detailed description of your changes, reference any related issues, and please follow the template provided. Gravitino's maintainers evaluate pull requests, offer feedback, and assist in merging project changes. -## Development setup - -Once you have cloned the [GitHub repository](https://github.com/datastrato/gravitino), see [how to build](/docs/how-to-build.md) for instructions on how to build, or you can use the provided docker images at [Datastrato's DockerHub repository](https://hub.docker.com/u/datastrato). +### The Review Process -To stop and start a local Gravitino server via ``bin/gravitino.sh start`` and ``bin/gravitino.sh stop`` in a Gravitino distribution, see [how to build](/docs/how-to-build.md) for more instructions. +For details on the review process, please refer to [MAINTAINERS.md](MAINTAINERS.md). ## Testing diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 983b568766b..5d28e44b439 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -1,6 +1,20 @@ # Governance Policy diff --git a/LICENSE b/LICENSE index 5d958349f2f..9d2c5a805eb 100644 --- a/LICENSE +++ b/LICENSE @@ -235,6 +235,7 @@ ./catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergExceptionMapper.java ./catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/DescribeIcebergSortOrderVisitor.java ./catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergTablePropertiesUtil.java + ./catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergHiveCachedClientPool.java ./clients/client-java/src/main/java/com/datastrato/gravitino/client/HTTPClient.java ./clients/client-java/src/main/java/com/datastrato/gravitino/client/RESTClient.java ./clients/client-java/src/test/java/com/datastrato/gravitino/client/TestHTTPClient.java @@ -246,6 +247,9 @@ ./clients/client-java/src/main/java/com/datastrato/gravitino/client/OAuth2ClientUtil.java ./gradlew + Apache Paimon + ./catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TypeUtils.java + Apache Hive ./catalogs/catalog-hive/src/test/resources/hive-schema-3.1.0.derby.sql diff --git a/LICENSE.bin b/LICENSE.bin index 60db5658127..b23d1bfefd4 100644 --- a/LICENSE.bin +++ b/LICENSE.bin @@ -355,6 +355,7 @@ XNIO API WildFly Confluent Kafka Streams Examples + Apache Arrow This product bundles various third-party components also under the Apache Software Foundation License 1.1 @@ -382,6 +383,7 @@ ParaNamer RE2/J ZSTD JNI + fsspec This product bundles various third-party components also under the MIT license @@ -393,6 +395,8 @@ Protocol Buffers Treelayout Kyligence/kylinpy + elarivie/pyReaderWriterLock + tkem/cachetools This product bundles various third-party components also under the Common Development and Distribution License 1.0 @@ -426,6 +430,7 @@ JTA AOP Alliance Repackaged OSGi Resource Locator + H2 Database Engine This product bundles various third-party components also under the Eclipse Public License 2.0 diff --git a/MAINTAINERS.md b/MAINTAINERS.md index ec967b2b752..ceb2505846a 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,6 +1,20 @@ This document lists the maintainers and contributors of the Project. diff --git a/NOTICE b/NOTICE index 6900d05a97d..3f221e49c50 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,8 @@ -Gravitino -Copyright 2023-2024 Datastrato Pvt Ltd +Apache Gravitino (incubating) +Copyright 2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). This product includes software developed at Datastrato (https://datastrato.ai). diff --git a/NOTICE.bin b/NOTICE.bin index 939c7f67ac1..5c63b8189ce 100644 --- a/NOTICE.bin +++ b/NOTICE.bin @@ -1,11 +1,11 @@ -Gravitino -Copyright 2023-2024 Datastrato Pvt Ltd +Apache Gravitino (incubating) +Copyright 2024 The Apache Software Foundation This product includes software developed at -Datastrato (https://datastrato.ai). +The Apache Software Foundation (http://www.apache.org/). This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +Datastrato (https://datastrato.ai). The Web UI also has a NOTICE file please see web/NOTICE for it's contents. diff --git a/README.md b/README.md index d961d63d99a..b7e3a9cc2c9 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,36 @@ -# Gravitino +# Apache Gravitino (incubating) -[![GitHub Actions Build](https://github.com/datastrato/gravitino/actions/workflows/build.yml/badge.svg)](https://github.com/datastrato/gravitino/actions/workflows/build.yml) -[![GitHub Actions Integration Test](https://github.com/datastrato/gravitino/actions/workflows/integration-test.yml/badge.svg)](https://github.com/datastrato/gravitino/actions/workflows/integration-test.yml) -[![License](https://img.shields.io/github/license/datastrato/gravitino)](https://github.com/datastrato/gravitino/blob/main/LICENSE) -[![Contributors](https://img.shields.io/github/contributors/datastrato/gravitino)](https://github.com/datastrato/gravitino/graphs/contributors) -[![Release](https://img.shields.io/github/v/release/datastrato/gravitino)](https://github.com/datastrato/gravitino/releases) -[![Open Issues](https://img.shields.io/github/issues-raw/datastrato/gravitino)](https://github.com/datastrato/gravitino/issues) -[![Last Committed](https://img.shields.io/github/last-commit/datastrato/gravitino)](https://github.com/datastrato/gravitino/commits/main/) +[![GitHub Actions Build](https://github.com/apache/gravitino/actions/workflows/build.yml/badge.svg)](https://github.com/apache/gravitino/actions/workflows/build.yml) +[![GitHub Actions Integration Test](https://github.com/apache/gravitino/actions/workflows/integration-test.yml/badge.svg)](https://github.com/apache/gravitino/actions/workflows/integration-test.yml) +[![License](https://img.shields.io/github/license/apache/gravitino)](https://github.com/apache/gravitino/blob/main/LICENSE) +[![Contributors](https://img.shields.io/github/contributors/apache/gravitino)](https://github.com/apache/gravitino/graphs/contributors) +[![Release](https://img.shields.io/github/v/release/apache/gravitino)](https://github.com/apache/gravitino/releases) +[![Open Issues](https://img.shields.io/github/issues-raw/apache/gravitino)](https://github.com/apache/gravitino/issues) +[![Last Committed](https://img.shields.io/github/last-commit/apache/gravitino)](https://github.com/apache/gravitino/commits/main/) [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8358/badge)](https://www.bestpractices.dev/projects/8358) ## Introduction -Gravitino is a high-performance, geo-distributed, and federated metadata lake. It manages the metadata directly in different sources, types, and regions. It also provides users with unified metadata access for data and AI assets. +Apache Gravitino is a high-performance, geo-distributed, and federated metadata lake. It manages the metadata directly in different sources, types, and regions. It also provides users with unified metadata access for data and AI assets. ![Gravitino Architecture](docs/assets/gravitino-architecture.png) @@ -27,7 +41,7 @@ Gravitino aims to provide several key features: * Security in one place, centralizing the security for different sources. * Built-in data management and data access management. -## Contributing to Gravitino +## Contributing to Apache Gravitino Gravitino is open source software available under the Apache 2.0 license. For information on how to contribute to Gravitino please see the [Contribution guidelines](CONTRIBUTING.md). @@ -35,7 +49,7 @@ Gravitino is open source software available under the Apache 2.0 license. For in You can find the latest Gravitino documentation in the [doc folder](docs). This README file only contains basic setup instructions. -## Building Gravitino +## Building Apache Gravitino You can build Gravitino using Gradle. Currently you can build Gravitino on Linux and macOS, Windows isn't supported. @@ -67,7 +81,7 @@ For the details of building and testing Gravitino, please see [How to build Grav ## Quick start -### Configure and start the Gravitino server +### Configure and start the Apache Gravitino server If you already have a binary distribution package, go to the directory of the decompressed package. @@ -86,7 +100,15 @@ To stop the Gravitino server, please run: ./bin/gravitino.sh stop ``` -### Using Trino with Gravitino +Alternatively, to run the Gravitino server in frontend, please run: + +```shell +./bin/gravitino.sh run +``` + +And press `CTRL+C` to stop the Gravitino server. + +### Using Trino with Apache Gravitino Gravitino provides a Trino connector to access the metadata in Gravitino. To use Trino with Gravitino, please follow the [trino-gravitino-connector doc](docs/trino-connector/index.md). @@ -100,4 +122,8 @@ Gravitino provides a Trino connector to access the metadata in Gravitino. To use Gravitino is under the Apache License Version 2.0, See the [LICENSE](LICENSE) for the details. -ApacheĀ®, Apache Hadoop®, Apache Hive™, Apache Iceberg™, Apache Kafka®, Apache Spark™, Apache Submarine™, Apache Thrift™ and Apache Zeppelin™ are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. +## ASF Incubator disclaimer + +Apache Gravitino is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF. + +ApacheĀ®, Apache Gravitino™, Apache Hadoop®, Apache Hive™, Apache Iceberg™, Apache Kafka®, Apache Spark™, Apache Submarine™, Apache Thrift™ and Apache Zeppelin™ are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. diff --git a/SECURITY.md b/SECURITY.md index e04052729c3..c24077d17c7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,6 +1,20 @@ # Security Policy diff --git a/api/build.gradle.kts b/api/build.gradle.kts index ede784c1c87..a9f2450863c 100644 --- a/api/build.gradle.kts +++ b/api/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ plugins { `maven-publish` diff --git a/api/src/main/java/com/datastrato/gravitino/Audit.java b/api/src/main/java/com/datastrato/gravitino/Audit.java index 4c3ee416b18..e727fd4615e 100644 --- a/api/src/main/java/com/datastrato/gravitino/Audit.java +++ b/api/src/main/java/com/datastrato/gravitino/Audit.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/api/src/main/java/com/datastrato/gravitino/Auditable.java b/api/src/main/java/com/datastrato/gravitino/Auditable.java index d2173293a13..f8e75b36e27 100644 --- a/api/src/main/java/com/datastrato/gravitino/Auditable.java +++ b/api/src/main/java/com/datastrato/gravitino/Auditable.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/api/src/main/java/com/datastrato/gravitino/Catalog.java b/api/src/main/java/com/datastrato/gravitino/Catalog.java index b62859a90c8..8c2d47efec0 100644 --- a/api/src/main/java/com/datastrato/gravitino/Catalog.java +++ b/api/src/main/java/com/datastrato/gravitino/Catalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; @@ -33,6 +47,24 @@ enum Type { UNSUPPORTED } + /** The cloud that the catalog is running on. Used by the catalog property `cloud.name`. */ + enum CloudName { + /** Amazon Web Services */ + AWS, + + /** Microsoft Azure */ + AZURE, + + /** Google Cloud Platform */ + GCP, + + /** Not running on cloud */ + ON_PREMISE, + + /** Other cloud providers */ + OTHER + } + /** * A reserved property to specify the package location of the catalog. The "package" is a string * of path to the folder where all the catalog related dependencies is located. The dependencies @@ -44,6 +76,18 @@ enum Type { */ String PROPERTY_PACKAGE = "package"; + /** + * The property to specify the cloud that the catalog is running on. The value should be one of + * the {@link CloudName}. + */ + String CLOUD_NAME = "cloud.name"; + + /** + * The property to specify the region code of the cloud that the catalog is running on. The value + * should be the region code of the cloud provider. + */ + String CLOUD_REGION_CODE = "cloud.region-code"; + /** @return The name of the catalog. */ String name(); diff --git a/api/src/main/java/com/datastrato/gravitino/CatalogChange.java b/api/src/main/java/com/datastrato/gravitino/CatalogChange.java index 7341bee13ac..4fd6055b841 100644 --- a/api/src/main/java/com/datastrato/gravitino/CatalogChange.java +++ b/api/src/main/java/com/datastrato/gravitino/CatalogChange.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/api/src/main/java/com/datastrato/gravitino/CatalogProvider.java b/api/src/main/java/com/datastrato/gravitino/CatalogProvider.java index eb682b68c4f..923e24979da 100644 --- a/api/src/main/java/com/datastrato/gravitino/CatalogProvider.java +++ b/api/src/main/java/com/datastrato/gravitino/CatalogProvider.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/api/src/main/java/com/datastrato/gravitino/MetadataObject.java b/api/src/main/java/com/datastrato/gravitino/MetadataObject.java index a166d7768bd..f9011d50cb7 100644 --- a/api/src/main/java/com/datastrato/gravitino/MetadataObject.java +++ b/api/src/main/java/com/datastrato/gravitino/MetadataObject.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; @@ -68,7 +82,13 @@ enum Type { * * @return The name of the object. */ - String fullName(); + default String fullName() { + if (parent() == null) { + return name(); + } else { + return parent() + "." + name(); + } + } /** * The type of the object. diff --git a/api/src/main/java/com/datastrato/gravitino/MetadataObjects.java b/api/src/main/java/com/datastrato/gravitino/MetadataObjects.java new file mode 100644 index 00000000000..c71c7b1ff8f --- /dev/null +++ b/api/src/main/java/com/datastrato/gravitino/MetadataObjects.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import java.util.List; +import org.apache.commons.lang3.StringUtils; + +/** The helper class for {@link MetadataObject}. */ +public class MetadataObjects { + + /** + * The reserved name for the metadata object. + * + *

It is used to represent the root metadata object of all metalakes. + */ + public static final String METADATA_OBJECT_RESERVED_NAME = "*"; + + private static final Splitter DOT_SPLITTER = Splitter.on('.'); + + private static final Joiner DOT_JOINER = Joiner.on('.'); + + private MetadataObjects() {} + + /** + * Create the metadata object with the given name, parent and type. + * + * @param parent The parent of the metadata object + * @param name The name of the metadata object + * @param type The type of the metadata object + * @return The created metadata object + */ + public static MetadataObject of(String parent, String name, MetadataObject.Type type) { + Preconditions.checkArgument(name != null, "Cannot create a metadata object with null name"); + Preconditions.checkArgument(type != null, "Cannot create a metadata object with no type"); + + return new MetadataObjectImpl(parent, name, type); + } + + /** + * Create the metadata object with the given names and type. + * + * @param names The names of the metadata object + * @param type The type of the metadata object + * @return The created metadata object + */ + public static MetadataObject of(List names, MetadataObject.Type type) { + Preconditions.checkArgument(names != null, "Cannot create a metadata object with null names"); + Preconditions.checkArgument(!names.isEmpty(), "Cannot create a metadata object with no names"); + Preconditions.checkArgument( + names.size() <= 4, + "Cannot create a metadata object with the name length which is greater than 4"); + Preconditions.checkArgument(type != null, "Cannot create a metadata object with no type"); + + Preconditions.checkArgument( + names.size() != 1 + || type == MetadataObject.Type.CATALOG + || type == MetadataObject.Type.METALAKE, + "If the length of names is 1, it must be the CATALOG or METALAKE type"); + + Preconditions.checkArgument( + names.size() != 2 || type == MetadataObject.Type.SCHEMA, + "If the length of names is 2, it must be the SCHEMA type"); + + Preconditions.checkArgument( + names.size() != 3 + || type == MetadataObject.Type.FILESET + || type == MetadataObject.Type.TABLE + || type == MetadataObject.Type.TOPIC, + "If the length of names is 3, it must be FILESET, TABLE or TOPIC"); + + Preconditions.checkArgument( + names.size() != 4 || type == MetadataObject.Type.COLUMN, + "If the length of names is 4, it must be COLUMN"); + + for (String name : names) { + checkName(name); + } + + return new MetadataObjectImpl(getParentFullName(names), getLastName(names), type); + } + + /** + * Parse the metadata object with the given full name and type. + * + * @param fullName The full name of the metadata object + * @param type The type of the metadata object + * @return The parsed metadata object + */ + public static MetadataObject parse(String fullName, MetadataObject.Type type) { + if (METADATA_OBJECT_RESERVED_NAME.equals(fullName)) { + if (type != MetadataObject.Type.METALAKE) { + throw new IllegalArgumentException("If metadata object isn't metalake, it can't be `*`"); + } + return new MetadataObjectImpl(null, METADATA_OBJECT_RESERVED_NAME, type); + } + + Preconditions.checkArgument( + StringUtils.isNotBlank(fullName), "Metadata object full name cannot be blank"); + + List parts = DOT_SPLITTER.splitToList(fullName); + + return MetadataObjects.of(parts, type); + } + + private static String getParentFullName(List names) { + if (names.size() <= 1) { + return null; + } + + return DOT_JOINER.join(names.subList(0, names.size() - 1)); + } + + private static String getLastName(List names) { + return names.get(names.size() - 1); + } + + private static void checkName(String name) { + Preconditions.checkArgument(name != null, "Cannot create a metadata object with null name"); + Preconditions.checkArgument( + !METADATA_OBJECT_RESERVED_NAME.equals(name), + "Cannot create a metadata object with `*` name."); + } + + /** The implementation of the {@link MetadataObject}. */ + public static class MetadataObjectImpl implements MetadataObject { + + private final String name; + + private final String parent; + + private final Type type; + + /** + * Create the metadata object with the given name, parent and type. + * + * @param parent The parent of the metadata object + * @param name The name of the metadata object + * @param type The type of the metadata object + */ + public MetadataObjectImpl(String parent, String name, Type type) { + this.parent = parent; + this.name = name; + this.type = type; + } + + @Override + public String name() { + return name; + } + + @Override + public String parent() { + return parent; + } + + @Override + public Type type() { + return type; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (!(o instanceof MetadataObjectImpl)) { + return false; + } + + MetadataObjectImpl that = (MetadataObjectImpl) o; + return java.util.Objects.equals(name, that.name) + && java.util.Objects.equals(parent, that.parent) + && type == that.type; + } + + @Override + public int hashCode() { + return java.util.Objects.hash(name, parent, type); + } + + @Override + public String toString() { + return "MetadataObject: [fullName=" + fullName() + "], [type=" + type + "]"; + } + } +} diff --git a/api/src/main/java/com/datastrato/gravitino/Metalake.java b/api/src/main/java/com/datastrato/gravitino/Metalake.java index c3f3fa64938..a7059761718 100644 --- a/api/src/main/java/com/datastrato/gravitino/Metalake.java +++ b/api/src/main/java/com/datastrato/gravitino/Metalake.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/api/src/main/java/com/datastrato/gravitino/MetalakeChange.java b/api/src/main/java/com/datastrato/gravitino/MetalakeChange.java index 3d787555a8c..33bd61f1d9f 100644 --- a/api/src/main/java/com/datastrato/gravitino/MetalakeChange.java +++ b/api/src/main/java/com/datastrato/gravitino/MetalakeChange.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/api/src/main/java/com/datastrato/gravitino/NameIdentifier.java b/api/src/main/java/com/datastrato/gravitino/NameIdentifier.java index b3c40bbdbcf..1fa9bd5a9a1 100644 --- a/api/src/main/java/com/datastrato/gravitino/NameIdentifier.java +++ b/api/src/main/java/com/datastrato/gravitino/NameIdentifier.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; @@ -50,152 +64,6 @@ public static NameIdentifier of(Namespace namespace, String name) { return new NameIdentifier(namespace, name); } - /** - * Create the metalake {@link NameIdentifier} with the given name. - * - * @param metalake The metalake name - * @return The created metalake {@link NameIdentifier} - */ - public static NameIdentifier ofMetalake(String metalake) { - return NameIdentifier.of(metalake); - } - - /** - * Create the catalog {@link NameIdentifier} with the given metalake and catalog name. - * - * @param metalake The metalake name - * @param catalog The catalog name - * @return The created catalog {@link NameIdentifier} - */ - public static NameIdentifier ofCatalog(String metalake, String catalog) { - return NameIdentifier.of(metalake, catalog); - } - - /** - * Create the schema {@link NameIdentifier} with the given metalake, catalog and schema name. - * - * @param metalake The metalake name - * @param catalog The catalog name - * @param schema The schema name - * @return The created schema {@link NameIdentifier} - */ - public static NameIdentifier ofSchema(String metalake, String catalog, String schema) { - return NameIdentifier.of(metalake, catalog, schema); - } - - /** - * Create the table {@link NameIdentifier} with the given metalake, catalog, schema and table - * name. - * - * @param metalake The metalake name - * @param catalog The catalog name - * @param schema The schema name - * @param table The table name - * @return The created table {@link NameIdentifier} - */ - public static NameIdentifier ofTable( - String metalake, String catalog, String schema, String table) { - return NameIdentifier.of(metalake, catalog, schema, table); - } - - /** - * Create the fileset {@link NameIdentifier} with the given metalake, catalog, schema and fileset - * name. - * - * @param metalake The metalake name - * @param catalog The catalog name - * @param schema The schema name - * @param fileset The fileset name - * @return The created fileset {@link NameIdentifier} - */ - public static NameIdentifier ofFileset( - String metalake, String catalog, String schema, String fileset) { - return NameIdentifier.of(metalake, catalog, schema, fileset); - } - - /** - * Create the topic {@link NameIdentifier} with the given metalake, catalog, schema and topic - * name. - * - * @param metalake The metalake name - * @param catalog The catalog name - * @param schema The schema name - * @param topic The topic name - * @return The created topic {@link NameIdentifier} - */ - public static NameIdentifier ofTopic( - String metalake, String catalog, String schema, String topic) { - return NameIdentifier.of(metalake, catalog, schema, topic); - } - - /** - * Check the given {@link NameIdentifier} is a metalake identifier. Throw an {@link - * IllegalNameIdentifierException} if it's not. - * - * @param ident The metalake {@link NameIdentifier} to check. - */ - public static void checkMetalake(NameIdentifier ident) { - // We don't have to check the name field of NameIdentifier, it's already checked when - // creating NameIdentifier object. - check(ident != null, "Metalake identifier must not be null"); - Namespace.checkMetalake(ident.namespace); - } - - /** - * Check the given {@link NameIdentifier} is a catalog identifier. Throw an {@link - * IllegalNameIdentifierException} if it's not. - * - * @param ident The catalog {@link NameIdentifier} to check. - */ - public static void checkCatalog(NameIdentifier ident) { - check(ident != null, "Catalog identifier must not be null"); - Namespace.checkCatalog(ident.namespace); - } - - /** - * Check the given {@link NameIdentifier} is a schema identifier. Throw an {@link - * IllegalNameIdentifierException} if it's not. - * - * @param ident The schema {@link NameIdentifier} to check. - */ - public static void checkSchema(NameIdentifier ident) { - check(ident != null, "Schema identifier must not be null"); - Namespace.checkSchema(ident.namespace); - } - - /** - * Check the given {@link NameIdentifier} is a table identifier. Throw an {@link - * IllegalNameIdentifierException} if it's not. - * - * @param ident The table {@link NameIdentifier} to check. - */ - public static void checkTable(NameIdentifier ident) { - check(ident != null, "Table identifier must not be null"); - Namespace.checkTable(ident.namespace); - } - - /** - * Check the given {@link NameIdentifier} is a fileset identifier. Throw an {@link - * IllegalNameIdentifierException} if it's not. - * - * @param ident The fileset {@link NameIdentifier} to check. - */ - public static void checkFileset(NameIdentifier ident) { - check(ident != null, "Fileset identifier must not be null"); - Namespace.checkFileset(ident.namespace); - } - - /** - * Check the given {@link NameIdentifier} is a topic identifier. Throw an {@link - * IllegalNameIdentifierException} if it's not. - * - * @param ident The topic {@link NameIdentifier} to check. - */ - public static void checkTopic(NameIdentifier ident) { - check(ident != null, "Topic identifier must not be null"); - Namespace.checkTopic(ident.namespace); - } - /** * Create a {@link NameIdentifier} from the given identifier string. * diff --git a/api/src/main/java/com/datastrato/gravitino/Namespace.java b/api/src/main/java/com/datastrato/gravitino/Namespace.java index ae469ff3d4c..d2527c963da 100644 --- a/api/src/main/java/com/datastrato/gravitino/Namespace.java +++ b/api/src/main/java/com/datastrato/gravitino/Namespace.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; @@ -51,150 +65,6 @@ public static Namespace of(String... levels) { return new Namespace(levels); } - /** - * Create a namespace for metalake. - * - * @return A namespace for metalake - */ - public static Namespace ofMetalake() { - return empty(); - } - - /** - * Create a namespace for catalog. - * - * @param metalake The metalake name - * @return A namespace for catalog - */ - public static Namespace ofCatalog(String metalake) { - return of(metalake); - } - - /** - * Create a namespace for schema. - * - * @param metalake The metalake name - * @param catalog The catalog name - * @return A namespace for schema - */ - public static Namespace ofSchema(String metalake, String catalog) { - return of(metalake, catalog); - } - - /** - * Create a namespace for table. - * - * @param metalake The metalake name - * @param catalog The catalog name - * @param schema The schema name - * @return A namespace for table - */ - public static Namespace ofTable(String metalake, String catalog, String schema) { - return of(metalake, catalog, schema); - } - - /** - * Create a namespace for fileset. - * - * @param metalake The metalake name - * @param catalog The catalog name - * @param schema The schema name - * @return A namespace for fileset - */ - public static Namespace ofFileset(String metalake, String catalog, String schema) { - return of(metalake, catalog, schema); - } - - /** - * Create a namespace for topic. - * - * @param metalake The metalake name - * @param catalog The catalog name - * @param schema The schema name - * @return A namespace for topic - */ - public static Namespace ofTopic(String metalake, String catalog, String schema) { - return of(metalake, catalog, schema); - } - - /** - * Check if the given metalake namespace is legal, throw an {@link IllegalNamespaceException} if - * it's illegal. - * - * @param namespace The metalake namespace - */ - public static void checkMetalake(Namespace namespace) { - check( - namespace != null && namespace.isEmpty(), - "Metalake namespace must be non-null and empty, the input namespace is %s", - namespace); - } - - /** - * Check if the given catalog namespace is legal, throw an {@link IllegalNamespaceException} if - * it's illegal. - * - * @param namespace The catalog namespace - */ - public static void checkCatalog(Namespace namespace) { - check( - namespace != null && namespace.length() == 1, - "Catalog namespace must be non-null and have 1 level, the input namespace is %s", - namespace); - } - - /** - * Check if the given schema namespace is legal, throw an {@link IllegalNamespaceException} if - * it's illegal. - * - * @param namespace The schema namespace - */ - public static void checkSchema(Namespace namespace) { - check( - namespace != null && namespace.length() == 2, - "Schema namespace must be non-null and have 2 levels, the input namespace is %s", - namespace); - } - - /** - * Check if the given table namespace is legal, throw an {@link IllegalNamespaceException} if it's - * illegal. - * - * @param namespace The table namespace - */ - public static void checkTable(Namespace namespace) { - check( - namespace != null && namespace.length() == 3, - "Table namespace must be non-null and have 3 levels, the input namespace is %s", - namespace); - } - - /** - * Check if the given fileset namespace is legal, throw an {@link IllegalNamespaceException} if - * it's illegal. - * - * @param namespace The fileset namespace - */ - public static void checkFileset(Namespace namespace) { - check( - namespace != null && namespace.length() == 3, - "Fileset namespace must be non-null and have 3 levels, the input namespace is %s", - namespace); - } - - /** - * Check if the given topic namespace is legal, throw an {@link IllegalNamespaceException} if it's - * illegal. - * - * @param namespace The topic namespace - */ - public static void checkTopic(Namespace namespace) { - check( - namespace != null && namespace.length() == 3, - "Topic namespace must be non-null and have 3 levels, the input namespace is %s", - namespace); - } - private Namespace(String[] levels) { this.levels = levels; } diff --git a/api/src/main/java/com/datastrato/gravitino/Schema.java b/api/src/main/java/com/datastrato/gravitino/Schema.java index b910ebdef3a..3e2e6e5f0aa 100644 --- a/api/src/main/java/com/datastrato/gravitino/Schema.java +++ b/api/src/main/java/com/datastrato/gravitino/Schema.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/api/src/main/java/com/datastrato/gravitino/SupportsCatalogs.java b/api/src/main/java/com/datastrato/gravitino/SupportsCatalogs.java index ea91a2c0cca..63e45cdfd14 100644 --- a/api/src/main/java/com/datastrato/gravitino/SupportsCatalogs.java +++ b/api/src/main/java/com/datastrato/gravitino/SupportsCatalogs.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; @@ -18,12 +32,12 @@ public interface SupportsCatalogs { /** - * List all catalogs in the metalake. + * List the name of all catalogs in the metalake. * - * @return The list of catalog's name identifiers. - * @throws NoSuchMetalakeException If the metalake with namespace does not exist. + * @return The list of catalog's names. + * @throws NoSuchMetalakeException If the metalake does not exist. */ - NameIdentifier[] listCatalogs() throws NoSuchMetalakeException; + String[] listCatalogs() throws NoSuchMetalakeException; /** * List all catalogs with their information in the metalake. diff --git a/api/src/main/java/com/datastrato/gravitino/SupportsMetalakes.java b/api/src/main/java/com/datastrato/gravitino/SupportsMetalakes.java index c8e4c10881f..340b421e05d 100644 --- a/api/src/main/java/com/datastrato/gravitino/SupportsMetalakes.java +++ b/api/src/main/java/com/datastrato/gravitino/SupportsMetalakes.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/api/src/main/java/com/datastrato/gravitino/annotation/DeveloperApi.java b/api/src/main/java/com/datastrato/gravitino/annotation/DeveloperApi.java index 2e725effea3..6da188f11d9 100644 --- a/api/src/main/java/com/datastrato/gravitino/annotation/DeveloperApi.java +++ b/api/src/main/java/com/datastrato/gravitino/annotation/DeveloperApi.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.annotation; diff --git a/api/src/main/java/com/datastrato/gravitino/annotation/Evolving.java b/api/src/main/java/com/datastrato/gravitino/annotation/Evolving.java index 6a1549850cc..129ab99c8fe 100644 --- a/api/src/main/java/com/datastrato/gravitino/annotation/Evolving.java +++ b/api/src/main/java/com/datastrato/gravitino/annotation/Evolving.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.annotation; diff --git a/api/src/main/java/com/datastrato/gravitino/annotation/Stable.java b/api/src/main/java/com/datastrato/gravitino/annotation/Stable.java index cdb0de47964..2b7d25d7f28 100644 --- a/api/src/main/java/com/datastrato/gravitino/annotation/Stable.java +++ b/api/src/main/java/com/datastrato/gravitino/annotation/Stable.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.annotation; diff --git a/api/src/main/java/com/datastrato/gravitino/annotation/Unstable.java b/api/src/main/java/com/datastrato/gravitino/annotation/Unstable.java index e1e7c2df841..3841f124959 100644 --- a/api/src/main/java/com/datastrato/gravitino/annotation/Unstable.java +++ b/api/src/main/java/com/datastrato/gravitino/annotation/Unstable.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.annotation; diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/Group.java b/api/src/main/java/com/datastrato/gravitino/authorization/Group.java index ff3e5ce7f42..8e741f99667 100644 --- a/api/src/main/java/com/datastrato/gravitino/authorization/Group.java +++ b/api/src/main/java/com/datastrato/gravitino/authorization/Group.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/Privilege.java b/api/src/main/java/com/datastrato/gravitino/authorization/Privilege.java index bc7f56b88a2..e89dd8718d6 100644 --- a/api/src/main/java/com/datastrato/gravitino/authorization/Privilege.java +++ b/api/src/main/java/com/datastrato/gravitino/authorization/Privilege.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/Privileges.java b/api/src/main/java/com/datastrato/gravitino/authorization/Privileges.java index 1bbe1254b1c..24df6352110 100644 --- a/api/src/main/java/com/datastrato/gravitino/authorization/Privileges.java +++ b/api/src/main/java/com/datastrato/gravitino/authorization/Privileges.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/Role.java b/api/src/main/java/com/datastrato/gravitino/authorization/Role.java index 02591035e4b..002f06b0bef 100644 --- a/api/src/main/java/com/datastrato/gravitino/authorization/Role.java +++ b/api/src/main/java/com/datastrato/gravitino/authorization/Role.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObject.java b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObject.java index 195e5e994c0..882faed29f6 100644 --- a/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObject.java +++ b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObject.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java index 7cf4fa8c765..fa1ded6eac7 100644 --- a/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java +++ b/api/src/main/java/com/datastrato/gravitino/authorization/SecurableObjects.java @@ -1,22 +1,36 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.MetadataObjects; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; /** The helper class for {@link SecurableObject}. */ public class SecurableObjects { - private static final Splitter DOT = Splitter.on('.'); + private static final Splitter DOT_SPLITTER = Splitter.on('.'); /** * Create the metalake {@link SecurableObject} with the given metalake name. @@ -51,7 +65,6 @@ public static SecurableObject ofCatalog(String catalog, List privileg */ public static SecurableObject ofSchema( SecurableObject catalog, String schema, List privileges) { - return of( SecurableObject.Type.SCHEMA, Lists.newArrayList(catalog.fullName(), schema), privileges); } @@ -66,7 +79,7 @@ public static SecurableObject ofSchema( */ public static SecurableObject ofTable( SecurableObject schema, String table, List privileges) { - List names = Lists.newArrayList(DOT.splitToList(schema.fullName())); + List names = Lists.newArrayList(DOT_SPLITTER.splitToList(schema.fullName())); names.add(table); return of(SecurableObject.Type.TABLE, names, privileges); } @@ -81,7 +94,7 @@ public static SecurableObject ofTable( */ public static SecurableObject ofTopic( SecurableObject schema, String topic, List privileges) { - List names = Lists.newArrayList(DOT.splitToList(schema.fullName())); + List names = Lists.newArrayList(DOT_SPLITTER.splitToList(schema.fullName())); names.add(topic); return of(SecurableObject.Type.TOPIC, names, privileges); } @@ -97,7 +110,7 @@ public static SecurableObject ofTopic( */ public static SecurableObject ofFileset( SecurableObject schema, String fileset, List privileges) { - List names = Lists.newArrayList(DOT.splitToList(schema.fullName())); + List names = Lists.newArrayList(DOT_SPLITTER.splitToList(schema.fullName())); names.add(fileset); return of(SecurableObject.Type.FILESET, names, privileges); } @@ -115,44 +128,16 @@ public static SecurableObject ofAllMetalakes(List privileges) { return new SecurableObjectImpl(null, "*", SecurableObject.Type.METALAKE, privileges); } - private static class SecurableObjectImpl implements SecurableObject { + private static class SecurableObjectImpl extends MetadataObjects.MetadataObjectImpl + implements SecurableObject { - private final String parent; - private final String name; - private final Type type; private List privileges; SecurableObjectImpl(String parent, String name, Type type, List privileges) { - this.parent = parent; - this.name = name; - this.type = type; + super(parent, name, type); this.privileges = ImmutableList.copyOf(privileges); } - @Override - public String parent() { - return parent; - } - - @Override - public String name() { - return name; - } - - @Override - public String fullName() { - if (parent != null) { - return parent + "." + name; - } else { - return name; - } - } - - @Override - public Type type() { - return type; - } - @Override public List privileges() { return privileges; @@ -160,7 +145,8 @@ public List privileges() { @Override public int hashCode() { - return Objects.hash(parent, name, type, privileges); + int result = super.hashCode(); + return Objects.hash(result, privileges); } @Override @@ -173,7 +159,7 @@ public String toString() { return "SecurableObject: [fullName=" + fullName() + "], [type=" - + type + + type() + "], [privileges=" + privilegesStr + "]"; @@ -181,15 +167,16 @@ public String toString() { @Override public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof SecurableObject)) { return false; } SecurableObject otherSecurableObject = (SecurableObject) other; - return Objects.equals(parent, otherSecurableObject.parent()) - && Objects.equals(name, otherSecurableObject.name()) - && Objects.equals(type, otherSecurableObject.type()) - && Objects.equals(privileges, otherSecurableObject.privileges()); + return super.equals(other) && Objects.equals(privileges, otherSecurableObject.privileges()); } } @@ -203,20 +190,9 @@ public boolean equals(Object other) { */ public static SecurableObject parse( String fullName, MetadataObject.Type type, List privileges) { - if ("*".equals(fullName)) { - if (type != MetadataObject.Type.METALAKE) { - throw new IllegalArgumentException("If securable object isn't metalake, it can't be `*`"); - } - return SecurableObjects.ofAllMetalakes(privileges); - } - - if (StringUtils.isBlank(fullName)) { - throw new IllegalArgumentException("securable object full name can't be blank"); - } - - List parts = DOT.splitToList(fullName); - - return SecurableObjects.of(type, parts, privileges); + MetadataObject metadataObject = MetadataObjects.parse(fullName, type); + return new SecurableObjectImpl( + metadataObject.parent(), metadataObject.name(), type, privileges); } /** @@ -229,68 +205,8 @@ public static SecurableObject parse( */ static SecurableObject of( MetadataObject.Type type, List names, List privileges) { - if (names == null) { - throw new IllegalArgumentException("Cannot create a securable object with null names"); - } - - if (names.isEmpty()) { - throw new IllegalArgumentException("Cannot create a securable object with no names"); - } - - if (type == null) { - throw new IllegalArgumentException("Cannot create a securable object with no type"); - } - - if (names.size() > 3) { - throw new IllegalArgumentException( - "Cannot create a securable object with the name length which is greater than 3"); - } - - if (names.size() == 1 - && type != MetadataObject.Type.CATALOG - && type != MetadataObject.Type.METALAKE) { - throw new IllegalArgumentException( - "If the length of names is 1, it must be the CATALOG or METALAKE type"); - } - - if (names.size() == 2 && type != SecurableObject.Type.SCHEMA) { - throw new IllegalArgumentException("If the length of names is 2, it must be the SCHEMA type"); - } - - if (names.size() == 3 - && type != MetadataObject.Type.FILESET - && type != MetadataObject.Type.TABLE - && type != MetadataObject.Type.TOPIC) { - throw new IllegalArgumentException( - "If the length of names is 3, it must be FILESET, TABLE or TOPIC"); - } - - for (String name : names) { - checkName(name); - } - - return new SecurableObjectImpl(getParentFullName(names), getLastName(names), type, privileges); - } - - private static String getParentFullName(List names) { - if (names.size() <= 1) { - return null; - } - - return String.join(".", names.subList(0, names.size() - 1)); - } - - private static String getLastName(List names) { - return names.get(names.size() - 1); - } - - private static void checkName(String name) { - if (name == null) { - throw new IllegalArgumentException("Cannot create a securable object with null name"); - } - - if ("*".equals(name)) { - throw new IllegalArgumentException("Cannot create a securable object with `*` name."); - } + MetadataObject metadataObject = MetadataObjects.of(names, type); + return new SecurableObjectImpl( + metadataObject.parent(), metadataObject.name(), type, privileges); } } diff --git a/api/src/main/java/com/datastrato/gravitino/authorization/User.java b/api/src/main/java/com/datastrato/gravitino/authorization/User.java index 32f57dd55a0..19e708553c8 100644 --- a/api/src/main/java/com/datastrato/gravitino/authorization/User.java +++ b/api/src/main/java/com/datastrato/gravitino/authorization/User.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/AlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/AlreadyExistsException.java index b1414f45c55..9179089d901 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/AlreadyExistsException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/AlreadyExistsException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/BadRequestException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/BadRequestException.java index 4876a3eda64..e8e83a077f5 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/BadRequestException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/BadRequestException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/CatalogAlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/CatalogAlreadyExistsException.java index 35625751920..e1458227968 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/CatalogAlreadyExistsException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/CatalogAlreadyExistsException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/FilesetAlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/FilesetAlreadyExistsException.java index cd9fb4db276..c7905727e72 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/FilesetAlreadyExistsException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/FilesetAlreadyExistsException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/GravitinoRuntimeException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/GravitinoRuntimeException.java index 8d0d62fe80f..3c2ad104442 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/GravitinoRuntimeException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/GravitinoRuntimeException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/GroupAlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/GroupAlreadyExistsException.java index edeb10b257d..64fb0459d6b 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/GroupAlreadyExistsException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/GroupAlreadyExistsException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/IllegalNameIdentifierException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/IllegalNameIdentifierException.java index a73ee5efb0f..43778600b9a 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/IllegalNameIdentifierException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/IllegalNameIdentifierException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/IllegalNamespaceException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/IllegalNamespaceException.java index 4d6d2d87e7b..9554cad82db 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/IllegalNamespaceException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/IllegalNamespaceException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/MetalakeAlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/MetalakeAlreadyExistsException.java index 22efd565926..37e38c6f499 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/MetalakeAlreadyExistsException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/MetalakeAlreadyExistsException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchCatalogException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchCatalogException.java index c80dc3e4c86..81aece2cea2 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchCatalogException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchCatalogException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchColumnException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchColumnException.java index 3f662b305ae..74bd57ccb44 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchColumnException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchColumnException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchEntityException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchEntityException.java index ecea805a5d2..3191d0a8e7a 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchEntityException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchEntityException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchFilesetException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchFilesetException.java index 660e5f3902b..f6603975467 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchFilesetException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchFilesetException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchGroupException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchGroupException.java index a9089233003..4324baae2e7 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchGroupException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchGroupException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchMetalakeException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchMetalakeException.java index bb9b311e541..a8fb58acf3a 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchMetalakeException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchMetalakeException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchPartitionException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchPartitionException.java index a10be950337..0ef7ab7f675 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchPartitionException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchPartitionException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchRoleException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchRoleException.java index 0ca9decfb65..d39f05493df 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchRoleException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchRoleException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchSchemaException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchSchemaException.java index df897dda33c..37bb74a7149 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchSchemaException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchSchemaException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchTableException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchTableException.java index 63a52a90cc8..0bd7cd7c0a7 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchTableException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchTableException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchTagException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchTagException.java index 2cdae1a0195..5fc2ff142a8 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchTagException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchTagException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchTopicException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchTopicException.java index c85b6c65c01..61ee70123c9 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchTopicException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchTopicException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchUserException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchUserException.java index 7d7043166e4..aac919d7697 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchUserException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchUserException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NonEmptyEntityException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NonEmptyEntityException.java index 5a51debeba8..4f35b5bae96 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NonEmptyEntityException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NonEmptyEntityException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NonEmptySchemaException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NonEmptySchemaException.java index af2516fe639..0a74185ba59 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NonEmptySchemaException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NonEmptySchemaException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NotFoundException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NotFoundException.java index 5a46e2c8fcb..4ffdcb90e66 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NotFoundException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NotFoundException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/PartitionAlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/PartitionAlreadyExistsException.java index 3f0cc96fb2b..2c9684c7b7d 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/PartitionAlreadyExistsException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/PartitionAlreadyExistsException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/RoleAlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/RoleAlreadyExistsException.java index d64dc7ff3ce..dcf7bd49d73 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/RoleAlreadyExistsException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/RoleAlreadyExistsException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/SchemaAlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/SchemaAlreadyExistsException.java index 45bf6cdb0e9..9041d45c8ae 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/SchemaAlreadyExistsException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/SchemaAlreadyExistsException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/TableAlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/TableAlreadyExistsException.java index 5c09ed806d8..21729e538d2 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/TableAlreadyExistsException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/TableAlreadyExistsException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/TagAlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/TagAlreadyExistsException.java index 385852a754d..bc1440a4640 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/TagAlreadyExistsException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/TagAlreadyExistsException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/TopicAlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/TopicAlreadyExistsException.java index 99198e58968..bf827aff34f 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/TopicAlreadyExistsException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/TopicAlreadyExistsException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/UnauthorizedException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/UnauthorizedException.java index e32ba66755a..ad1646709c4 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/UnauthorizedException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/UnauthorizedException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/UserAlreadyExistsException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/UserAlreadyExistsException.java index 322897235cc..eedae84459f 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/UserAlreadyExistsException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/UserAlreadyExistsException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.exceptions; diff --git a/api/src/main/java/com/datastrato/gravitino/file/Fileset.java b/api/src/main/java/com/datastrato/gravitino/file/Fileset.java index 6c5bc4441c7..57e7aa1cf13 100644 --- a/api/src/main/java/com/datastrato/gravitino/file/Fileset.java +++ b/api/src/main/java/com/datastrato/gravitino/file/Fileset.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.file; diff --git a/api/src/main/java/com/datastrato/gravitino/file/FilesetCatalog.java b/api/src/main/java/com/datastrato/gravitino/file/FilesetCatalog.java index 389c32afad6..164965524df 100644 --- a/api/src/main/java/com/datastrato/gravitino/file/FilesetCatalog.java +++ b/api/src/main/java/com/datastrato/gravitino/file/FilesetCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.file; diff --git a/api/src/main/java/com/datastrato/gravitino/file/FilesetChange.java b/api/src/main/java/com/datastrato/gravitino/file/FilesetChange.java index 2e76ac34461..6fef3398901 100644 --- a/api/src/main/java/com/datastrato/gravitino/file/FilesetChange.java +++ b/api/src/main/java/com/datastrato/gravitino/file/FilesetChange.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.file; diff --git a/api/src/main/java/com/datastrato/gravitino/messaging/DataLayout.java b/api/src/main/java/com/datastrato/gravitino/messaging/DataLayout.java index 86c5ae3e0d4..e57775fbd07 100644 --- a/api/src/main/java/com/datastrato/gravitino/messaging/DataLayout.java +++ b/api/src/main/java/com/datastrato/gravitino/messaging/DataLayout.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.messaging; diff --git a/api/src/main/java/com/datastrato/gravitino/messaging/Topic.java b/api/src/main/java/com/datastrato/gravitino/messaging/Topic.java index aa356c09c8e..db9c6486264 100644 --- a/api/src/main/java/com/datastrato/gravitino/messaging/Topic.java +++ b/api/src/main/java/com/datastrato/gravitino/messaging/Topic.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.messaging; diff --git a/api/src/main/java/com/datastrato/gravitino/messaging/TopicCatalog.java b/api/src/main/java/com/datastrato/gravitino/messaging/TopicCatalog.java index b2979697f6e..67e4ed09946 100644 --- a/api/src/main/java/com/datastrato/gravitino/messaging/TopicCatalog.java +++ b/api/src/main/java/com/datastrato/gravitino/messaging/TopicCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.messaging; diff --git a/api/src/main/java/com/datastrato/gravitino/messaging/TopicChange.java b/api/src/main/java/com/datastrato/gravitino/messaging/TopicChange.java index e07189c7d70..e4bbdcd78f1 100644 --- a/api/src/main/java/com/datastrato/gravitino/messaging/TopicChange.java +++ b/api/src/main/java/com/datastrato/gravitino/messaging/TopicChange.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.messaging; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/Column.java b/api/src/main/java/com/datastrato/gravitino/rel/Column.java index a02015d0dcf..db08b368f19 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/Column.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/Column.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/SupportsPartitions.java b/api/src/main/java/com/datastrato/gravitino/rel/SupportsPartitions.java index 9cc3c1b97d8..adcafc648ba 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/SupportsPartitions.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/SupportsPartitions.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/Table.java b/api/src/main/java/com/datastrato/gravitino/rel/Table.java index 4efd91a5b63..d53d3d3e514 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/Table.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/Table.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/expressions/FunctionExpression.java b/api/src/main/java/com/datastrato/gravitino/rel/expressions/FunctionExpression.java index 988abece9cd..5b833fcffbb 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/expressions/FunctionExpression.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/expressions/FunctionExpression.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.expressions; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/expressions/UnparsedExpression.java b/api/src/main/java/com/datastrato/gravitino/rel/expressions/UnparsedExpression.java index a4925f8c9a6..8f9a649fa52 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/expressions/UnparsedExpression.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/expressions/UnparsedExpression.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.expressions; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/expressions/distributions/Distribution.java b/api/src/main/java/com/datastrato/gravitino/rel/expressions/distributions/Distribution.java index 7cfc6eea6c5..9f56bd538c6 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/expressions/distributions/Distribution.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/expressions/distributions/Distribution.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.expressions.distributions; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/expressions/distributions/Distributions.java b/api/src/main/java/com/datastrato/gravitino/rel/expressions/distributions/Distributions.java index 6e44d2342ba..d09b1db6774 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/expressions/distributions/Distributions.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/expressions/distributions/Distributions.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.expressions.distributions; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/expressions/distributions/Strategy.java b/api/src/main/java/com/datastrato/gravitino/rel/expressions/distributions/Strategy.java index 0ad68850ec1..50b0b214b96 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/expressions/distributions/Strategy.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/expressions/distributions/Strategy.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.expressions.distributions; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/expressions/literals/Literal.java b/api/src/main/java/com/datastrato/gravitino/rel/expressions/literals/Literal.java index bf362f022ec..2006351ad88 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/expressions/literals/Literal.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/expressions/literals/Literal.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.expressions.literals; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/expressions/literals/Literals.java b/api/src/main/java/com/datastrato/gravitino/rel/expressions/literals/Literals.java index a1c8e2fd609..4db95c86a5f 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/expressions/literals/Literals.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/expressions/literals/Literals.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.expressions.literals; @@ -50,6 +64,16 @@ public static LiteralImpl byteLiteral(Byte value) { return of(value, Types.ByteType.get()); } + /** + * Creates an unsigned byte type literal with the given value. + * + * @param value the unsigned byte literal value + * @return a new {@link Literal} instance + */ + public static LiteralImpl unsignedByteLiteral(Short value) { + return of(value, Types.ByteType.unsigned()); + } + /** * Creates a short type literal with the given value. * @@ -60,6 +84,16 @@ public static LiteralImpl shortLiteral(Short value) { return of(value, Types.ShortType.get()); } + /** + * Creates an unsigned short type literal with the given value. + * + * @param value the unsigned short literal value + * @return a new {@link Literal} instance + */ + public static LiteralImpl unsignedShortLiteral(Integer value) { + return of(value, Types.ShortType.unsigned()); + } + /** * Creates an integer type literal with the given value. * @@ -70,6 +104,16 @@ public static LiteralImpl integerLiteral(Integer value) { return of(value, Types.IntegerType.get()); } + /** + * Creates an unsigned integer type literal with the given value. + * + * @param value the unsigned integer literal value + * @return a new {@link Literal} instance + */ + public static LiteralImpl unsignedIntegerLiteral(Long value) { + return of(value, Types.IntegerType.unsigned()); + } + /** * Creates a long type literal with the given value. * @@ -80,6 +124,16 @@ public static LiteralImpl longLiteral(Long value) { return of(value, Types.LongType.get()); } + /** + * Creates an unsigned long type literal with the given value. + * + * @param value the unsigned long literal value + * @return a new {@link Literal} instance + */ + public static LiteralImpl unsignedLongLiteral(Decimal value) { + return of(value, Types.LongType.unsigned()); + } + /** * Creates a float type literal with the given value. * diff --git a/api/src/main/java/com/datastrato/gravitino/rel/expressions/sorts/SortOrders.java b/api/src/main/java/com/datastrato/gravitino/rel/expressions/sorts/SortOrders.java index c07bcdaeab9..e2727f7f6a4 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/expressions/sorts/SortOrders.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/expressions/sorts/SortOrders.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.expressions.sorts; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/expressions/transforms/Transforms.java b/api/src/main/java/com/datastrato/gravitino/rel/expressions/transforms/Transforms.java index ecc98a76ad0..e1a821ab253 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/expressions/transforms/Transforms.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/expressions/transforms/Transforms.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.expressions.transforms; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/indexes/Index.java b/api/src/main/java/com/datastrato/gravitino/rel/indexes/Index.java index 93142919161..b71c91bc5fb 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/indexes/Index.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/indexes/Index.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.indexes; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java b/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java index 531c0290d05..6ed0a1ad20b 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.indexes; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/partitions/IdentityPartition.java b/api/src/main/java/com/datastrato/gravitino/rel/partitions/IdentityPartition.java index 81b4291cd0a..9cf0026bc49 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/partitions/IdentityPartition.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/partitions/IdentityPartition.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.partitions; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/partitions/ListPartition.java b/api/src/main/java/com/datastrato/gravitino/rel/partitions/ListPartition.java index 5a9a9f549dd..528556deb64 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/partitions/ListPartition.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/partitions/ListPartition.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.partitions; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/partitions/Partition.java b/api/src/main/java/com/datastrato/gravitino/rel/partitions/Partition.java index 7eb496743b2..f8255d745ae 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/partitions/Partition.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/partitions/Partition.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.partitions; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/partitions/Partitions.java b/api/src/main/java/com/datastrato/gravitino/rel/partitions/Partitions.java index bbd49d10291..fb340555efa 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/partitions/Partitions.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/partitions/Partitions.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.partitions; @@ -24,7 +38,7 @@ public class Partitions { * @param properties The properties of the partition. * @return The created partition. */ - public static Partition range( + public static RangePartition range( String name, Literal upper, Literal lower, Map properties) { return new RangePartitionImpl(name, upper, lower, properties); } @@ -37,7 +51,8 @@ public static Partition range( * @param properties The properties of the partition. * @return The created partition. */ - public static Partition list(String name, Literal[][] lists, Map properties) { + public static ListPartition list( + String name, Literal[][] lists, Map properties) { return new ListPartitionImpl(name, lists, properties); } @@ -52,7 +67,7 @@ public static Partition list(String name, Literal[][] lists, Map[] values, Map properties) { return new IdentityPartitionImpl(name, fieldNames, values, properties); } @@ -64,7 +79,7 @@ public static Partition identity( * @param values The values of the identity partition. * @return The created partition. */ - public static Partition identity(String[][] fieldNames, Literal[] values) { + public static IdentityPartition identity(String[][] fieldNames, Literal[] values) { return identity(null, fieldNames, values, null); } diff --git a/api/src/main/java/com/datastrato/gravitino/rel/partitions/RangePartition.java b/api/src/main/java/com/datastrato/gravitino/rel/partitions/RangePartition.java index fbdfb67b810..df19dad4e1c 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/partitions/RangePartition.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/partitions/RangePartition.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.partitions; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/types/Decimal.java b/api/src/main/java/com/datastrato/gravitino/rel/types/Decimal.java index b9964d1b598..702e85f4719 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/types/Decimal.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/types/Decimal.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.types; diff --git a/api/src/main/java/com/datastrato/gravitino/rel/types/Type.java b/api/src/main/java/com/datastrato/gravitino/rel/types/Type.java index ede539a871b..f6b1145a1f2 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/types/Type.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/types/Type.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.types; @@ -101,7 +115,19 @@ abstract class IntervalType extends PrimitiveType {} abstract class ComplexType implements Type {} /** The base type of all integral types. */ - abstract class IntegralType extends NumericType {} + abstract class IntegralType extends NumericType { + private final boolean signed; + + /** @param signed or unsigned of the integer type. */ + public IntegralType(boolean signed) { + this.signed = signed; + } + + /** @return True if the integer type has signed, false otherwise. */ + public boolean signed() { + return signed; + } + } /** The base type of all fractional types. */ abstract class FractionType extends NumericType {} diff --git a/api/src/main/java/com/datastrato/gravitino/rel/types/Types.java b/api/src/main/java/com/datastrato/gravitino/rel/types/Types.java index 8848f423de5..ca805a304b3 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/types/Types.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/types/Types.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel.types; @@ -58,14 +72,22 @@ public String simpleString() { /** The byte type in Gravitino. */ public static class ByteType extends Type.IntegralType { - private static final ByteType INSTANCE = new ByteType(); + private static final ByteType INSTANCE = new ByteType(true); + private static final ByteType UNSIGNED_INSTANCE = new ByteType(false); + + /** @return The singleton instance of unsigned byte type */ + public static ByteType unsigned() { + return UNSIGNED_INSTANCE; + } /** @return The singleton instance of {@link ByteType}. */ public static ByteType get() { return INSTANCE; } - private ByteType() {} + private ByteType(boolean signed) { + super(signed); + } @Override public Name name() { @@ -74,20 +96,28 @@ public Name name() { @Override public String simpleString() { - return "byte"; + return signed() ? "byte" : "byte unsigned"; } } /** The short type in Gravitino. */ public static class ShortType extends Type.IntegralType { - private static final ShortType INSTANCE = new ShortType(); + private static final ShortType INSTANCE = new ShortType(true); + private static final ShortType UNSIGNED_INSTANCE = new ShortType(false); + + /** @return The singleton instance of unsigned short type */ + public static ShortType unsigned() { + return UNSIGNED_INSTANCE; + } /** @return The singleton instance of {@link ShortType}. */ public static ShortType get() { return INSTANCE; } - private ShortType() {} + private ShortType(boolean signed) { + super(signed); + } @Override public Name name() { @@ -96,20 +126,28 @@ public Name name() { @Override public String simpleString() { - return "short"; + return signed() ? "short" : "short unsigned"; } } /** The integer type in Gravitino. */ public static class IntegerType extends Type.IntegralType { - private static final IntegerType INSTANCE = new IntegerType(); + private static final IntegerType INSTANCE = new IntegerType(true); + private static final IntegerType UNSIGNED_INSTANCE = new IntegerType(false); + + /** @return The singleton instance of unsigned integer type */ + public static IntegerType unsigned() { + return UNSIGNED_INSTANCE; + } /** @return The singleton instance of {@link IntegerType}. */ public static IntegerType get() { return INSTANCE; } - private IntegerType() {} + private IntegerType(boolean signed) { + super(signed); + } @Override public Name name() { @@ -118,20 +156,28 @@ public Name name() { @Override public String simpleString() { - return "integer"; + return signed() ? "integer" : "integer unsigned"; } } /** The long type in Gravitino. */ public static class LongType extends Type.IntegralType { - private static final LongType INSTANCE = new LongType(); + private static final LongType INSTANCE = new LongType(true); + private static final LongType UNSIGNED_INSTANCE = new LongType(false); + + /** @return The singleton instance of unsigned long type */ + public static LongType unsigned() { + return UNSIGNED_INSTANCE; + } /** @return The singleton instance of {@link LongType}. */ public static LongType get() { return INSTANCE; } - private LongType() {} + private LongType(boolean signed) { + super(signed); + } @Override public Name name() { @@ -140,7 +186,7 @@ public Name name() { @Override public String simpleString() { - return "long"; + return signed() ? "long" : "long unsigned"; } } diff --git a/api/src/main/java/com/datastrato/gravitino/tag/SupportsTags.java b/api/src/main/java/com/datastrato/gravitino/tag/SupportsTags.java index 3259029faec..d386f6741ae 100644 --- a/api/src/main/java/com/datastrato/gravitino/tag/SupportsTags.java +++ b/api/src/main/java/com/datastrato/gravitino/tag/SupportsTags.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.tag; diff --git a/api/src/main/java/com/datastrato/gravitino/tag/Tag.java b/api/src/main/java/com/datastrato/gravitino/tag/Tag.java index c1bb51d286c..e5a38729d76 100644 --- a/api/src/main/java/com/datastrato/gravitino/tag/Tag.java +++ b/api/src/main/java/com/datastrato/gravitino/tag/Tag.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.tag; diff --git a/api/src/main/java/com/datastrato/gravitino/tag/TagChange.java b/api/src/main/java/com/datastrato/gravitino/tag/TagChange.java index 1f5c30fca56..2ce96526a53 100644 --- a/api/src/main/java/com/datastrato/gravitino/tag/TagChange.java +++ b/api/src/main/java/com/datastrato/gravitino/tag/TagChange.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.tag; diff --git a/api/src/main/java/com/datastrato/gravitino/tag/TagOperations.java b/api/src/main/java/com/datastrato/gravitino/tag/TagOperations.java index 4f4d3534d44..593a7561dda 100644 --- a/api/src/main/java/com/datastrato/gravitino/tag/TagOperations.java +++ b/api/src/main/java/com/datastrato/gravitino/tag/TagOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.tag; diff --git a/api/src/test/java/com/datastrato/gravitino/TestCatalogChange.java b/api/src/test/java/com/datastrato/gravitino/TestCatalogChange.java index 7261f9b803d..97fee914e42 100644 --- a/api/src/test/java/com/datastrato/gravitino/TestCatalogChange.java +++ b/api/src/test/java/com/datastrato/gravitino/TestCatalogChange.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/api/src/test/java/com/datastrato/gravitino/TestMetaLakeChange.java b/api/src/test/java/com/datastrato/gravitino/TestMetaLakeChange.java index 6e42d2fcadc..642364125f6 100644 --- a/api/src/test/java/com/datastrato/gravitino/TestMetaLakeChange.java +++ b/api/src/test/java/com/datastrato/gravitino/TestMetaLakeChange.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/api/src/test/java/com/datastrato/gravitino/TestNameIdentifier.java b/api/src/test/java/com/datastrato/gravitino/TestNameIdentifier.java index aefb79bf8eb..1f36a4052c5 100644 --- a/api/src/test/java/com/datastrato/gravitino/TestNameIdentifier.java +++ b/api/src/test/java/com/datastrato/gravitino/TestNameIdentifier.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; @@ -10,8 +24,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import com.datastrato.gravitino.exceptions.IllegalNameIdentifierException; -import com.datastrato.gravitino.exceptions.IllegalNamespaceException; import org.junit.jupiter.api.Test; public class TestNameIdentifier { @@ -92,34 +104,4 @@ public void testToString() { assertEquals("a", id1.toString()); assertEquals("a.b.c", id2.toString()); } - - @Test - public void testCheckNameIdentifier() { - NameIdentifier abc = NameIdentifier.of("a", "b", "c"); - NameIdentifier abcd = NameIdentifier.of("a", "b", "c", "d"); - - // Test metalake - assertThrows(IllegalNameIdentifierException.class, () -> NameIdentifier.checkMetalake(null)); - Throwable excep = - assertThrows(IllegalNamespaceException.class, () -> NameIdentifier.checkMetalake(abc)); - assertTrue(excep.getMessage().contains("Metalake namespace must be non-null and empty")); - - // test catalog - assertThrows(IllegalNameIdentifierException.class, () -> NameIdentifier.checkCatalog(null)); - Throwable excep1 = - assertThrows(IllegalNamespaceException.class, () -> NameIdentifier.checkCatalog(abc)); - assertTrue(excep1.getMessage().contains("Catalog namespace must be non-null and have 1 level")); - - // test schema - assertThrows(IllegalNameIdentifierException.class, () -> NameIdentifier.checkSchema(null)); - Throwable excep2 = - assertThrows(IllegalNamespaceException.class, () -> NameIdentifier.checkSchema(abcd)); - assertTrue(excep2.getMessage().contains("Schema namespace must be non-null and have 2 levels")); - - // test table - assertThrows(IllegalNameIdentifierException.class, () -> NameIdentifier.checkTable(null)); - Throwable excep3 = - assertThrows(IllegalNamespaceException.class, () -> NameIdentifier.checkTable(abc)); - assertTrue(excep3.getMessage().contains("Table namespace must be non-null and have 3 levels")); - } } diff --git a/api/src/test/java/com/datastrato/gravitino/TestNamespace.java b/api/src/test/java/com/datastrato/gravitino/TestNamespace.java index 8b12b75c22f..b0ee47caaad 100644 --- a/api/src/test/java/com/datastrato/gravitino/TestNamespace.java +++ b/api/src/test/java/com/datastrato/gravitino/TestNamespace.java @@ -1,10 +1,23 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; -import com.datastrato.gravitino.exceptions.IllegalNamespaceException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -33,39 +46,4 @@ public void testCreateNamespace() { Assertions.assertThrows(IllegalArgumentException.class, () -> Namespace.of("a", null, "c")); Assertions.assertThrows(IllegalArgumentException.class, () -> Namespace.of("a", "", "c")); } - - @Test - public void testCheckNamespace() { - Namespace a = Namespace.of("a"); - Namespace ab = Namespace.of("a", "b"); - Namespace abcd = Namespace.of("a", "b", "c", "d"); - - // Test metalake - Assertions.assertThrows(IllegalNamespaceException.class, () -> Namespace.checkMetalake(null)); - Throwable excep = - Assertions.assertThrows(IllegalNamespaceException.class, () -> Namespace.checkMetalake(ab)); - Assertions.assertTrue( - excep.getMessage().contains("Metalake namespace must be non-null and empty")); - - // Test catalog - Assertions.assertThrows(IllegalNamespaceException.class, () -> Namespace.checkCatalog(null)); - Throwable excep1 = - Assertions.assertThrows(IllegalNamespaceException.class, () -> Namespace.checkCatalog(ab)); - Assertions.assertTrue( - excep1.getMessage().contains("Catalog namespace must be non-null and have 1 level")); - - // Test schema - Assertions.assertThrows(IllegalNamespaceException.class, () -> Namespace.checkSchema(null)); - Throwable excep2 = - Assertions.assertThrows(IllegalNamespaceException.class, () -> Namespace.checkSchema(a)); - Assertions.assertTrue( - excep2.getMessage().contains("Schema namespace must be non-null and have 2 levels")); - - // Test table - Assertions.assertThrows(IllegalNamespaceException.class, () -> Namespace.checkTable(null)); - Throwable excep3 = - Assertions.assertThrows(IllegalNamespaceException.class, () -> Namespace.checkTable(abcd)); - Assertions.assertTrue( - excep3.getMessage().contains("Table namespace must be non-null and have 3 levels")); - } } diff --git a/api/src/test/java/com/datastrato/gravitino/TestSchemaChange.java b/api/src/test/java/com/datastrato/gravitino/TestSchemaChange.java index 5dd7de7f462..0a5da73f268 100644 --- a/api/src/test/java/com/datastrato/gravitino/TestSchemaChange.java +++ b/api/src/test/java/com/datastrato/gravitino/TestSchemaChange.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/api/src/test/java/com/datastrato/gravitino/TestTableChange.java b/api/src/test/java/com/datastrato/gravitino/TestTableChange.java index f7e22f711a3..a477aac13f2 100644 --- a/api/src/test/java/com/datastrato/gravitino/TestTableChange.java +++ b/api/src/test/java/com/datastrato/gravitino/TestTableChange.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/api/src/test/java/com/datastrato/gravitino/authorization/TestSecurableObjects.java b/api/src/test/java/com/datastrato/gravitino/authorization/TestSecurableObjects.java index 894d7df6420..f3ccb8cdb0f 100644 --- a/api/src/test/java/com/datastrato/gravitino/authorization/TestSecurableObjects.java +++ b/api/src/test/java/com/datastrato/gravitino/authorization/TestSecurableObjects.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; diff --git a/api/src/test/java/com/datastrato/gravitino/rel/TestColumn.java b/api/src/test/java/com/datastrato/gravitino/rel/TestColumn.java index ae416003aeb..1680783686c 100644 --- a/api/src/test/java/com/datastrato/gravitino/rel/TestColumn.java +++ b/api/src/test/java/com/datastrato/gravitino/rel/TestColumn.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel; diff --git a/api/src/test/java/com/datastrato/gravitino/rel/TestDistribution.java b/api/src/test/java/com/datastrato/gravitino/rel/TestDistribution.java index 0a4f7717598..8b29b94fb28 100644 --- a/api/src/test/java/com/datastrato/gravitino/rel/TestDistribution.java +++ b/api/src/test/java/com/datastrato/gravitino/rel/TestDistribution.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel; diff --git a/api/src/test/java/com/datastrato/gravitino/rel/TestLiteral.java b/api/src/test/java/com/datastrato/gravitino/rel/TestLiteral.java index a9641a8c9a1..94b3a6a4430 100644 --- a/api/src/test/java/com/datastrato/gravitino/rel/TestLiteral.java +++ b/api/src/test/java/com/datastrato/gravitino/rel/TestLiteral.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel; @@ -16,6 +30,10 @@ import static com.datastrato.gravitino.rel.expressions.literals.Literals.stringLiteral; import static com.datastrato.gravitino.rel.expressions.literals.Literals.timeLiteral; import static com.datastrato.gravitino.rel.expressions.literals.Literals.timestampLiteral; +import static com.datastrato.gravitino.rel.expressions.literals.Literals.unsignedByteLiteral; +import static com.datastrato.gravitino.rel.expressions.literals.Literals.unsignedIntegerLiteral; +import static com.datastrato.gravitino.rel.expressions.literals.Literals.unsignedLongLiteral; +import static com.datastrato.gravitino.rel.expressions.literals.Literals.unsignedShortLiteral; import com.datastrato.gravitino.rel.expressions.literals.Literal; import com.datastrato.gravitino.rel.expressions.literals.Literals; @@ -40,18 +58,34 @@ public void testLiterals() { Assertions.assertEquals((byte) 1, literal.value()); Assertions.assertEquals(Types.ByteType.get(), literal.dataType()); + literal = unsignedByteLiteral(Short.valueOf("1")); + Assertions.assertEquals((short) 1, literal.value()); + Assertions.assertEquals(Types.ByteType.unsigned(), literal.dataType()); + literal = shortLiteral(Short.valueOf("1")); Assertions.assertEquals((short) 1, literal.value()); Assertions.assertEquals(Types.ShortType.get(), literal.dataType()); + literal = unsignedShortLiteral(Integer.valueOf("1")); + Assertions.assertEquals(1, literal.value()); + Assertions.assertEquals(Types.ShortType.unsigned(), literal.dataType()); + literal = integerLiteral(Integer.valueOf("1")); Assertions.assertEquals(1, literal.value()); Assertions.assertEquals(Types.IntegerType.get(), literal.dataType()); + literal = unsignedIntegerLiteral(Long.valueOf("1")); + Assertions.assertEquals(1L, literal.value()); + Assertions.assertEquals(Types.IntegerType.unsigned(), literal.dataType()); + literal = longLiteral(Long.valueOf("1")); Assertions.assertEquals(1L, literal.value()); Assertions.assertEquals(Types.LongType.get(), literal.dataType()); + literal = unsignedLongLiteral(Decimal.of("1")); + Assertions.assertEquals(Decimal.of("1"), literal.value()); + Assertions.assertEquals(Types.LongType.unsigned(), literal.dataType()); + literal = floatLiteral(Float.valueOf("1.234")); Assertions.assertEquals(1.234f, literal.value()); Assertions.assertEquals(Types.FloatType.get(), literal.dataType()); diff --git a/api/src/test/java/com/datastrato/gravitino/rel/TestPartitions.java b/api/src/test/java/com/datastrato/gravitino/rel/TestPartitions.java index b8eba965fc4..a0e56d6204b 100644 --- a/api/src/test/java/com/datastrato/gravitino/rel/TestPartitions.java +++ b/api/src/test/java/com/datastrato/gravitino/rel/TestPartitions.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel; diff --git a/api/src/test/java/com/datastrato/gravitino/rel/TestSortOrder.java b/api/src/test/java/com/datastrato/gravitino/rel/TestSortOrder.java index 1bbf9d9aa75..5a36f6bd135 100644 --- a/api/src/test/java/com/datastrato/gravitino/rel/TestSortOrder.java +++ b/api/src/test/java/com/datastrato/gravitino/rel/TestSortOrder.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel; diff --git a/api/src/test/java/com/datastrato/gravitino/rel/TestTransforms.java b/api/src/test/java/com/datastrato/gravitino/rel/TestTransforms.java index a52fc21076d..2793ff6c173 100644 --- a/api/src/test/java/com/datastrato/gravitino/rel/TestTransforms.java +++ b/api/src/test/java/com/datastrato/gravitino/rel/TestTransforms.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel; diff --git a/api/src/test/java/com/datastrato/gravitino/rel/TestTypes.java b/api/src/test/java/com/datastrato/gravitino/rel/TestTypes.java index f623451867d..5674c2c91e9 100644 --- a/api/src/test/java/com/datastrato/gravitino/rel/TestTypes.java +++ b/api/src/test/java/com/datastrato/gravitino/rel/TestTypes.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rel; @@ -26,24 +40,44 @@ public void testPrimitiveTypes() { Types.ByteType byteType = Types.ByteType.get(); Assertions.assertEquals(Type.Name.BYTE, byteType.name()); - Assertions.assertSame(byteType, Types.ByteType.get()); Assertions.assertEquals("byte", byteType.simpleString()); + Types.ByteType unsignByteType = Types.ByteType.unsigned(); + Assertions.assertEquals(Type.Name.BYTE, unsignByteType.name()); + Assertions.assertFalse(unsignByteType.signed()); + Assertions.assertEquals("byte unsigned", unsignByteType.simpleString()); + Types.ShortType shortType = Types.ShortType.get(); Assertions.assertEquals(Type.Name.SHORT, shortType.name()); Assertions.assertSame(shortType, Types.ShortType.get()); Assertions.assertEquals("short", shortType.simpleString()); + Types.ShortType unsignShortType = Types.ShortType.unsigned(); + Assertions.assertEquals(Type.Name.SHORT, unsignShortType.name()); + Assertions.assertFalse(unsignShortType.signed()); + Assertions.assertEquals("short unsigned", unsignShortType.simpleString()); + Types.IntegerType integerType = Types.IntegerType.get(); Assertions.assertEquals(Type.Name.INTEGER, integerType.name()); Assertions.assertSame(integerType, Types.IntegerType.get()); Assertions.assertEquals("integer", integerType.simpleString()); + Types.IntegerType unsignIntegerType = Types.IntegerType.unsigned(); + Assertions.assertEquals(Type.Name.INTEGER, unsignIntegerType.name()); + Assertions.assertFalse(unsignIntegerType.signed()); + Assertions.assertEquals("integer unsigned", unsignIntegerType.simpleString()); + Types.LongType longType = Types.LongType.get(); Assertions.assertEquals(Type.Name.LONG, longType.name()); Assertions.assertSame(longType, Types.LongType.get()); Assertions.assertEquals("long", longType.simpleString()); + Types.LongType unsignLongType = Types.LongType.unsigned(); + Assertions.assertEquals(Type.Name.LONG, unsignLongType.name()); + Assertions.assertFalse(unsignLongType.signed()); + Assertions.assertNotSame(unsignLongType, Types.LongType.get()); + Assertions.assertEquals("long unsigned", unsignLongType.simpleString()); + Types.FloatType floatType = Types.FloatType.get(); Assertions.assertEquals(Type.Name.FLOAT, floatType.name()); Assertions.assertSame(floatType, Types.FloatType.get()); diff --git a/api/src/test/resources/log4j2.properties b/api/src/test/resources/log4j2.properties index b9e706603a0..f3155e4c00b 100644 --- a/api/src/test/resources/log4j2.properties +++ b/api/src/test/resources/log4j2.properties @@ -1,6 +1,20 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # Set to debug or trace if log4j initialization is failing diff --git a/bin/gravitino.sh b/bin/gravitino.sh index 065aa1603be..4e54abc41ab 100755 --- a/bin/gravitino.sh +++ b/bin/gravitino.sh @@ -1,11 +1,25 @@ #!/bin/bash # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # #set -ex USAGE="-e Usage: bin/gravitino.sh [--config ]\n\t - {start|stop|restart|status}" + {start|run|stop|restart|status}" if [[ "$1" == "--config" ]]; then shift @@ -110,6 +124,10 @@ function start() { check_process_status } +function run() { + ${JAVA_RUNNER} ${JAVA_OPTS} ${GRAVITINO_DEBUG_OPTS} -cp ${GRAVITINO_CLASSPATH} ${GRAVITINO_SERVER_NAME} +} + function stop() { local pid @@ -164,6 +182,9 @@ case "${1}" in start) start ;; + run) + run + ;; stop) stop ;; diff --git a/build.gradle.kts b/build.gradle.kts index 36affb2c006..32e13d72cb9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ import com.github.gradle.node.NodeExtension import com.github.gradle.node.NodePlugin @@ -10,12 +24,10 @@ import com.github.jk1.license.render.InventoryHtmlReportRenderer import com.github.jk1.license.render.ReportRenderer import com.github.vlsi.gradle.dsl.configureEach import net.ltgt.gradle.errorprone.errorprone -import org.gradle.api.tasks.testing.Test import org.gradle.api.tasks.testing.logging.TestExceptionFormat import org.gradle.internal.hash.ChecksumService import org.gradle.internal.os.OperatingSystem import org.gradle.kotlin.dsl.support.serviceOf -import java.io.File import java.io.IOException import java.util.Locale @@ -186,9 +198,9 @@ allprojects { } param.useJUnitPlatform { - val DOCKER_IT_TEST = project.rootProject.extra["docker_it_test"] as? Boolean ?: false - if (!DOCKER_IT_TEST) { - excludeTags("gravitino-docker-it") + val dockerTest = project.rootProject.extra["dockerTest"] as? Boolean ?: false + if (!dockerTest) { + excludeTags("gravitino-docker-test") } } } @@ -442,9 +454,6 @@ subprojects { } tasks.rat { - substringMatcher("DS", "Datastrato", "Copyright 2023 Datastrato Pvt Ltd.") - substringMatcher("DS", "Datastrato", "Copyright 2024 Datastrato Pvt Ltd.") - approvedLicense("Datastrato") approvedLicense("Apache License Version 2.0") // Set input directory to that of the root project instead of the CWD. This @@ -452,7 +461,7 @@ tasks.rat { inputDir.set(project.rootDir) val exclusions = mutableListOf( - // Ignore files we track but do not need headers + // Ignore files we track but do not need full headers "**/.github/**/*", "dev/docker/**/*.xml", "dev/docker/**/*.conf", @@ -461,12 +470,13 @@ tasks.rat { "**/licenses/*.txt", "**/licenses/*.md", "integration-test/**", + "docs/**/*.md", "web/.**", "web/next-env.d.ts", "web/dist/**/*", "web/node_modules/**/*", "web/src/lib/utils/axios/**/*", - "web/src/lib/enums/httpEnum.ts", + "web/src/lib/enums/httpEnum.js", "web/src/types/axios.d.ts", "web/yarn.lock", "web/package-lock.json", @@ -641,6 +651,7 @@ tasks { dependsOn( ":catalogs:catalog-hive:copyLibAndConfig", ":catalogs:catalog-lakehouse-iceberg:copyLibAndConfig", + ":catalogs:catalog-lakehouse-paimon:copyLibAndConfig", ":catalogs:catalog-jdbc-doris:copyLibAndConfig", ":catalogs:catalog-jdbc-mysql:copyLibAndConfig", ":catalogs:catalog-jdbc-postgresql:copyLibAndConfig", @@ -656,7 +667,7 @@ tasks { apply(plugin = "com.dorongold.task-tree") -project.extra["docker_it_test"] = false +project.extra["dockerTest"] = false project.extra["dockerRunning"] = false project.extra["macDockerConnector"] = false project.extra["isOrbStack"] = false @@ -675,13 +686,15 @@ fun printDockerCheckInfo() { val macDockerConnector = project.extra["macDockerConnector"] as? Boolean ?: false val isOrbStack = project.extra["isOrbStack"] as? Boolean ?: false - if (OperatingSystem.current().isMacOsX() && + if (extra["skipDockerTests"].toString().toBoolean()) { + project.extra["dockerTest"] = false + } else if (OperatingSystem.current().isMacOsX() && dockerRunning && (macDockerConnector || isOrbStack) ) { - project.extra["docker_it_test"] = true + project.extra["dockerTest"] = true } else if (OperatingSystem.current().isLinux() && dockerRunning) { - project.extra["docker_it_test"] = true + project.extra["dockerTest"] = true } println("------------------ Check Docker environment ---------------------") @@ -691,11 +704,11 @@ fun printDockerCheckInfo() { println("OrbStack status ................................................. [${if (dockerRunning && isOrbStack) "yes" else "no"}]") } - val docker_it_test = project.extra["docker_it_test"] as? Boolean ?: false - if (!docker_it_test) { - println("Run test cases without `gravitino-docker-it` tag ................ [$testMode test]") + val dockerTest = project.extra["dockerTest"] as? Boolean ?: false + if (dockerTest) { + println("Using Docker container to run all tests. [$testMode test]") } else { - println("Using Gravitino IT Docker container to run all integration tests. [$testMode test]") + println("Run test cases without `gravitino-docker-test` tag ................ [$testMode test]") } println("-----------------------------------------------------------------") @@ -728,8 +741,8 @@ fun printMacDockerTip() { } fun checkMacDockerConnector() { - if (OperatingSystem.current().isLinux()) { - // Linux does not require the use of `docker-connector` + if (!OperatingSystem.current().isMacOsX()) { + // Only MacOs requires the use of `docker-connector` return } @@ -764,7 +777,7 @@ fun checkDockerStatus() { } fun checkOrbStackStatus() { - if (OperatingSystem.current().isLinux()) { + if (!OperatingSystem.current().isMacOsX()) { return } diff --git a/catalogs/build.gradle.kts b/catalogs/build.gradle.kts index 2d4a7344247..043fbfec673 100644 --- a/catalogs/build.gradle.kts +++ b/catalogs/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ tasks.all { diff --git a/catalogs/bundled-catalog/build.gradle.kts b/catalogs/bundled-catalog/build.gradle.kts index 23ef280c6c2..11534bcda66 100644 --- a/catalogs/bundled-catalog/build.gradle.kts +++ b/catalogs/bundled-catalog/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar @@ -16,6 +30,7 @@ dependencies { implementation(project(":catalogs:catalog-jdbc-mysql")) implementation(project(":catalogs:catalog-jdbc-postgresql")) implementation(project(":catalogs:catalog-lakehouse-iceberg")) + implementation(project(":catalogs:catalog-lakehouse-paimon")) implementation(project(":core")) implementation(libs.slf4j.api) } @@ -80,6 +95,7 @@ tasks.jar { tasks.compileJava { dependsOn(":catalogs:catalog-jdbc-postgresql:runtimeJars") dependsOn(":catalogs:catalog-lakehouse-iceberg:runtimeJars") + dependsOn(":catalogs:catalog-lakehouse-paimon:runtimeJars") dependsOn(":catalogs:catalog-jdbc-mysql:runtimeJars") dependsOn(":catalogs:catalog-hive:runtimeJars") dependsOn(":catalogs:catalog-hadoop:runtimeJars") diff --git a/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/common/ClassProvider.java b/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/common/ClassProvider.java index 3e923dfac0d..9cba50cba4c 100644 --- a/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/common/ClassProvider.java +++ b/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/common/ClassProvider.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.common; diff --git a/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/property/PropertyConverter.java b/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/property/PropertyConverter.java index 30619f832cf..bb850ba8944 100644 --- a/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/property/PropertyConverter.java +++ b/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/property/PropertyConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.property; diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 09f47dd075f..ccdf7c99614 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ description = "catalog-hadoop" diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalog.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalog.java index d548b4e548c..04f25fa3bce 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalog.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalog.java @@ -1,10 +1,24 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop; -import com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig; +import com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig; import com.datastrato.gravitino.connector.BaseCatalog; import com.datastrato.gravitino.connector.CatalogOperations; import com.datastrato.gravitino.connector.PropertiesMetadata; diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogCapability.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogCapability.java index d063a04fa4f..cf8f7324ef9 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogCapability.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogCapability.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop; diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 283b03c3b33..4d60687a109 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop; @@ -15,8 +29,8 @@ import com.datastrato.gravitino.Schema; import com.datastrato.gravitino.SchemaChange; import com.datastrato.gravitino.StringIdentifier; -import com.datastrato.gravitino.catalog.hadoop.kerberos.AuthenticationConfig; -import com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosClient; +import com.datastrato.gravitino.catalog.hadoop.authentication.AuthenticationConfig; +import com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosClient; import com.datastrato.gravitino.connector.CatalogInfo; import com.datastrato.gravitino.connector.CatalogOperations; import com.datastrato.gravitino.connector.HasPropertyMetadata; diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 37a5700e57c..44c0fef8291 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -1,11 +1,25 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop; -import com.datastrato.gravitino.catalog.hadoop.kerberos.AuthenticationConfig; -import com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig; +import com.datastrato.gravitino.catalog.hadoop.authentication.AuthenticationConfig; +import com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig; import com.datastrato.gravitino.connector.BaseCatalogPropertiesMetadata; import com.datastrato.gravitino.connector.PropertyEntry; import com.google.common.collect.ImmutableMap; diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopFileset.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopFileset.java index 5d5d9b939dd..ad1ffbaf176 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopFileset.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopFileset.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop; diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java index 218d69b30f6..23a7276daaa 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop; diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopProxyPlugin.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopProxyPlugin.java index 471f6a165cd..75650ab6a78 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopProxyPlugin.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopProxyPlugin.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop; diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopSchema.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopSchema.java index 7d6043b555e..d831303d60c 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopSchema.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopSchema.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop; diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java index 5cf23913a69..bac0c94dcd1 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop; diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/AuthenticationConfig.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/authentication/AuthenticationConfig.java similarity index 63% rename from catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/AuthenticationConfig.java rename to catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/authentication/AuthenticationConfig.java index 8d9549c57b6..5a25e3519ab 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/AuthenticationConfig.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/authentication/AuthenticationConfig.java @@ -1,13 +1,26 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package com.datastrato.gravitino.catalog.hadoop.kerberos; - -import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.DEFAULT_IMPERSONATION_ENABLE; +package com.datastrato.gravitino.catalog.hadoop.authentication; import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig; import com.datastrato.gravitino.config.ConfigBuilder; import com.datastrato.gravitino.config.ConfigConstants; import com.datastrato.gravitino.config.ConfigEntry; @@ -29,17 +42,18 @@ public AuthenticationConfig(Map properties) { public static final ConfigEntry AUTH_TYPE_ENTRY = new ConfigBuilder(AUTH_TYPE_KEY) - .doc("The type of authentication for Hadoop catalog, currently we only support Kerberos") + .doc( + "The type of authentication for Hadoop catalog, currently we only support simple and Kerberos") .version(ConfigConstants.VERSION_0_5_1) .stringConf() - .create(); + .createWithDefault("simple"); public static final ConfigEntry ENABLE_IMPERSONATION_ENTRY = new ConfigBuilder(IMPERSONATION_ENABLE_KEY) .doc("Whether to enable impersonation for the Hadoop catalog") .version(ConfigConstants.VERSION_0_5_1) .booleanConf() - .createWithDefault(DEFAULT_IMPERSONATION_ENABLE); + .createWithDefault(KerberosConfig.DEFAULT_IMPERSONATION_ENABLE); public String getAuthType() { return get(AUTH_TYPE_ENTRY); @@ -58,7 +72,7 @@ public boolean isImpersonationEnabled() { "Whether to enable impersonation for the Hadoop catalog", false, true, - DEFAULT_IMPERSONATION_ENABLE, + KerberosConfig.DEFAULT_IMPERSONATION_ENABLE, false, false)) .put( diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/FetchFileUtils.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/authentication/kerberos/FetchFileUtils.java similarity index 61% rename from catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/FetchFileUtils.java rename to catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/authentication/kerberos/FetchFileUtils.java index 30f75687f6e..ab24b4f36f1 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/FetchFileUtils.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/authentication/kerberos/FetchFileUtils.java @@ -1,8 +1,22 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package com.datastrato.gravitino.catalog.hadoop.kerberos; +package com.datastrato.gravitino.catalog.hadoop.authentication.kerberos; import java.io.File; import java.io.IOException; diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/KerberosClient.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/authentication/kerberos/KerberosClient.java similarity index 81% rename from catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/KerberosClient.java rename to catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/authentication/kerberos/KerberosClient.java index b9aece8b87c..8a5eff13b22 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/KerberosClient.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/authentication/kerberos/KerberosClient.java @@ -1,9 +1,23 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package com.datastrato.gravitino.catalog.hadoop.kerberos; +package com.datastrato.gravitino.catalog.hadoop.authentication.kerberos; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/KerberosConfig.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/authentication/kerberos/KerberosConfig.java similarity index 79% rename from catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/KerberosConfig.java rename to catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/authentication/kerberos/KerberosConfig.java index 60f5b8469c5..ed77746fe0c 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/KerberosConfig.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/authentication/kerberos/KerberosConfig.java @@ -1,10 +1,25 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package com.datastrato.gravitino.catalog.hadoop.kerberos; +package com.datastrato.gravitino.catalog.hadoop.authentication.kerberos; +import com.datastrato.gravitino.catalog.hadoop.authentication.AuthenticationConfig; import com.datastrato.gravitino.config.ConfigBuilder; import com.datastrato.gravitino.config.ConfigConstants; import com.datastrato.gravitino.config.ConfigEntry; diff --git a/catalogs/catalog-hadoop/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider b/catalogs/catalog-hadoop/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider index 455c78cd331..1fba0f9ed35 100644 --- a/catalogs/catalog-hadoop/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider +++ b/catalogs/catalog-hadoop/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider @@ -1,5 +1,19 @@ # -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # com.datastrato.gravitino.catalog.hadoop.HadoopCatalog diff --git a/catalogs/catalog-hadoop/src/main/resources/core-site.xml.template b/catalogs/catalog-hadoop/src/main/resources/core-site.xml.template index 48da733c1c3..efa7db5adb6 100644 --- a/catalogs/catalog-hadoop/src/main/resources/core-site.xml.template +++ b/catalogs/catalog-hadoop/src/main/resources/core-site.xml.template @@ -1,6 +1,20 @@ diff --git a/catalogs/catalog-hadoop/src/main/resources/hadoop.conf b/catalogs/catalog-hadoop/src/main/resources/hadoop.conf index 1ed75088d96..605c9b43a27 100644 --- a/catalogs/catalog-hadoop/src/main/resources/hadoop.conf +++ b/catalogs/catalog-hadoop/src/main/resources/hadoop.conf @@ -1,6 +1,20 @@ # -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # This file holds common properties for Hadoop catalog. All the created Hadoop catalog will diff --git a/catalogs/catalog-hadoop/src/main/resources/hdfs-site.xml.template b/catalogs/catalog-hadoop/src/main/resources/hdfs-site.xml.template index 7ad37de8b5c..a47ec72adb9 100644 --- a/catalogs/catalog-hadoop/src/main/resources/hdfs-site.xml.template +++ b/catalogs/catalog-hadoop/src/main/resources/hdfs-site.xml.template @@ -1,6 +1,20 @@ diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index c4d624050c0..6fa41cd9521 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -1,13 +1,27 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop; import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_KV_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; import static com.datastrato.gravitino.Configs.ENTITY_KV_STORE; import static com.datastrato.gravitino.Configs.ENTITY_STORE; -import static com.datastrato.gravitino.Configs.ENTRY_KV_ROCKSDB_BACKEND_PATH; import static com.datastrato.gravitino.Configs.STORE_DELETE_AFTER_TIME; import static com.datastrato.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME; import static com.datastrato.gravitino.catalog.hadoop.HadoopCatalog.CATALOG_PROPERTIES_META; @@ -35,6 +49,7 @@ import com.datastrato.gravitino.file.FilesetChange; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.storage.RandomIdGenerator; +import com.datastrato.gravitino.utils.NameIdentifierUtil; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import java.io.IOException; @@ -106,9 +121,9 @@ public static void setUp() { Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv"); Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE); Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto"); - Mockito.when(config.get(ENTRY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(ROCKS_DB_STORE_PATH); + Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(ROCKS_DB_STORE_PATH); - Assertions.assertEquals(ROCKS_DB_STORE_PATH, config.get(ENTRY_KV_ROCKSDB_BACKEND_PATH)); + Assertions.assertEquals(ROCKS_DB_STORE_PATH, config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)); Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); @@ -221,13 +236,13 @@ public void testLoadSchema() throws IOException { String comment = "comment15"; String catalogPath = TEST_ROOT_PATH + "/" + "catalog15"; Schema schema = createSchema(name, comment, catalogPath, null); - NameIdentifier schema16 = NameIdentifier.ofSchema("m1", "c1", "schema16"); + NameIdentifier schema16 = NameIdentifierUtil.ofSchema("m1", "c1", "schema16"); Assertions.assertEquals(name, schema.name()); try (HadoopCatalogOperations ops = new HadoopCatalogOperations(store)) { ops.initialize(Maps.newHashMap(), null, HADOOP_PROPERTIES_METADATA); - Schema schema1 = ops.loadSchema(NameIdentifier.ofSchema("m1", "c1", name)); + Schema schema1 = ops.loadSchema(NameIdentifierUtil.ofSchema("m1", "c1", name)); Assertions.assertEquals(name, schema1.name()); Assertions.assertEquals(comment, schema1.comment()); @@ -254,8 +269,8 @@ public void testListSchema() throws IOException { Set idents = Arrays.stream(ops.listSchemas(Namespace.of("m1", "c1"))).collect(Collectors.toSet()); Assertions.assertTrue(idents.size() >= 2); - Assertions.assertTrue(idents.contains(NameIdentifier.ofSchema("m1", "c1", name))); - Assertions.assertTrue(idents.contains(NameIdentifier.ofSchema("m1", "c1", name1))); + Assertions.assertTrue(idents.contains(NameIdentifierUtil.ofSchema("m1", "c1", name))); + Assertions.assertTrue(idents.contains(NameIdentifierUtil.ofSchema("m1", "c1", name1))); } } @@ -269,7 +284,7 @@ public void testAlterSchema() throws IOException { try (HadoopCatalogOperations ops = new HadoopCatalogOperations(store)) { ops.initialize(Maps.newHashMap(), null, HADOOP_PROPERTIES_METADATA); - Schema schema1 = ops.loadSchema(NameIdentifier.ofSchema("m1", "c1", name)); + Schema schema1 = ops.loadSchema(NameIdentifierUtil.ofSchema("m1", "c1", name)); Assertions.assertEquals(name, schema1.name()); Assertions.assertEquals(comment, schema1.comment()); @@ -279,26 +294,27 @@ public void testAlterSchema() throws IOException { String newKey = "k1"; String newValue = "v1"; SchemaChange setProperty = SchemaChange.setProperty(newKey, newValue); - Schema schema2 = ops.alterSchema(NameIdentifier.ofSchema("m1", "c1", name), setProperty); + Schema schema2 = ops.alterSchema(NameIdentifierUtil.ofSchema("m1", "c1", name), setProperty); Assertions.assertEquals(name, schema2.name()); Assertions.assertEquals(comment, schema2.comment()); Map props2 = schema2.properties(); Assertions.assertTrue(props2.containsKey(newKey)); Assertions.assertEquals(newValue, props2.get(newKey)); - Schema schema3 = ops.loadSchema(NameIdentifier.ofSchema("m1", "c1", name)); + Schema schema3 = ops.loadSchema(NameIdentifierUtil.ofSchema("m1", "c1", name)); Map props3 = schema3.properties(); Assertions.assertTrue(props3.containsKey(newKey)); Assertions.assertEquals(newValue, props3.get(newKey)); SchemaChange removeProperty = SchemaChange.removeProperty(newKey); - Schema schema4 = ops.alterSchema(NameIdentifier.ofSchema("m1", "c1", name), removeProperty); + Schema schema4 = + ops.alterSchema(NameIdentifierUtil.ofSchema("m1", "c1", name), removeProperty); Assertions.assertEquals(name, schema4.name()); Assertions.assertEquals(comment, schema4.comment()); Map props4 = schema4.properties(); Assertions.assertFalse(props4.containsKey(newKey)); - Schema schema5 = ops.loadSchema(NameIdentifier.ofSchema("m1", "c1", name)); + Schema schema5 = ops.loadSchema(NameIdentifierUtil.ofSchema("m1", "c1", name)); Map props5 = schema5.properties(); Assertions.assertFalse(props5.containsKey(newKey)); } @@ -311,7 +327,7 @@ public void testDropSchema() throws IOException { String catalogPath = TEST_ROOT_PATH + "/" + "catalog20"; Schema schema = createSchema(name, comment, catalogPath, null); Assertions.assertEquals(name, schema.name()); - NameIdentifier id = NameIdentifier.ofSchema("m1", "c1", name); + NameIdentifier id = NameIdentifierUtil.ofSchema("m1", "c1", name); try (HadoopCatalogOperations ops = new HadoopCatalogOperations(store)) { ops.initialize( @@ -369,7 +385,7 @@ public void testCreateLoadAndDeleteFilesetWithLocations( catalogProps.put(HadoopCatalogPropertiesMetadata.LOCATION, catalogPath); } - NameIdentifier schemaIdent = NameIdentifier.ofSchema("m1", "c1", schemaName); + NameIdentifier schemaIdent = NameIdentifierUtil.ofSchema("m1", "c1", schemaName); try (HadoopCatalogOperations ops = new HadoopCatalogOperations(store)) { ops.initialize(catalogProps, null, HADOOP_PROPERTIES_METADATA); if (!ops.schemaExists(schemaIdent)) { @@ -491,7 +507,7 @@ public void testRenameFileset( catalogProps.put(HadoopCatalogPropertiesMetadata.LOCATION, catalogPath); } - NameIdentifier schemaIdent = NameIdentifier.ofSchema("m1", "c1", schemaName); + NameIdentifier schemaIdent = NameIdentifierUtil.ofSchema("m1", "c1", schemaName); try (HadoopCatalogOperations ops = new HadoopCatalogOperations(store)) { ops.initialize(catalogProps, null, HADOOP_PROPERTIES_METADATA); if (!ops.schemaExists(schemaIdent)) { @@ -858,7 +874,7 @@ private Schema createSchema(String name, String comment, String catalogPath, Str try (HadoopCatalogOperations ops = new HadoopCatalogOperations(store)) { ops.initialize(props, null, HADOOP_PROPERTIES_METADATA); - NameIdentifier schemaIdent = NameIdentifier.ofSchema("m1", "c1", name); + NameIdentifier schemaIdent = NameIdentifierUtil.ofSchema("m1", "c1", name); Map schemaProps = Maps.newHashMap(); StringIdentifier stringId = StringIdentifier.fromId(idGenerator.nextId()); schemaProps = Maps.newHashMap(StringIdentifier.newPropertiesWithId(stringId, schemaProps)); diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java index a28407b7f08..4a5f6c66098 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop.integration.test; @@ -27,7 +41,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class HDFSKerberosIT { private static final Logger LOG = LoggerFactory.getLogger(HDFSKerberosIT.class); diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java index e79432f209d..310785f96b1 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop.integration.test; @@ -35,7 +49,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class HadoopCatalogIT extends AbstractIT { private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogIT.class); private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); @@ -258,8 +272,7 @@ public void testExternalFileset() throws IOException { void testNameSpec() { String illegalName = "/%~?*"; - NameIdentifier nameIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, illegalName); + NameIdentifier nameIdentifier = NameIdentifier.of(schemaName, illegalName); Assertions.assertThrows( NoSuchFilesetException.class, () -> catalog.asFilesetCatalog().loadFileset(nameIdentifier)); @@ -282,9 +295,7 @@ public void testLoadFileset() throws IOException { // test load fileset Fileset loadFileset = - catalog - .asFilesetCatalog() - .loadFileset(NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName)); + catalog.asFilesetCatalog().loadFileset(NameIdentifier.of(schemaName, filesetName)); Assertions.assertEquals(fileset.name(), loadFileset.name(), "fileset should be loaded"); Assertions.assertEquals(fileset.comment(), loadFileset.comment(), "comment should be loaded"); Assertions.assertEquals(fileset.type(), loadFileset.type(), "type should be loaded"); @@ -298,10 +309,7 @@ public void testLoadFileset() throws IOException { // test load a fileset that not exist Assertions.assertThrows( NoSuchFilesetException.class, - () -> - catalog - .asFilesetCatalog() - .loadFileset(NameIdentifier.of(metalakeName, catalogName, schemaName, "not_exist")), + () -> catalog.asFilesetCatalog().loadFileset(NameIdentifier.of(schemaName, "not_exist")), "Should throw NoSuchFilesetException when fileset does not exist"); } @@ -320,16 +328,12 @@ public void testDropManagedFileset() throws IOException { // drop fileset boolean dropped = - catalog - .asFilesetCatalog() - .dropFileset(NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName)); + catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(schemaName, filesetName)); Assertions.assertTrue(dropped, "fileset should be dropped"); // verify fileset is dropped Assertions.assertFalse( - catalog - .asFilesetCatalog() - .filesetExists(NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName)), + catalog.asFilesetCatalog().filesetExists(NameIdentifier.of(schemaName, filesetName)), "fileset should not be exists"); Assertions.assertFalse( hdfs.exists(new Path(storageLocation)), "storage location should be dropped"); @@ -351,16 +355,12 @@ public void testDropExternalFileset() throws IOException { // drop fileset boolean dropped = - catalog - .asFilesetCatalog() - .dropFileset(NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName)); + catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(schemaName, filesetName)); Assertions.assertTrue(dropped, "fileset should be dropped"); // verify fileset is dropped Assertions.assertFalse( - catalog - .asFilesetCatalog() - .filesetExists(NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName)), + catalog.asFilesetCatalog().filesetExists(NameIdentifier.of(schemaName, filesetName)), "fileset should not be exists"); Assertions.assertTrue( hdfs.exists(new Path(storageLocation)), "storage location should not be dropped"); @@ -374,9 +374,7 @@ public void testListFilesets() throws IOException { // test no fileset exists NameIdentifier[] nameIdentifiers = - catalog - .asFilesetCatalog() - .listFilesets(Namespace.ofFileset(metalakeName, catalogName, schemaName)); + catalog.asFilesetCatalog().listFilesets(Namespace.of(schemaName)); Assertions.assertEquals(0, nameIdentifiers.length, "should have no fileset"); // create fileset1 @@ -407,9 +405,7 @@ public void testListFilesets() throws IOException { // list filesets NameIdentifier[] nameIdentifiers1 = - catalog - .asFilesetCatalog() - .listFilesets(Namespace.ofFileset(metalakeName, catalogName, schemaName)); + catalog.asFilesetCatalog().listFilesets(Namespace.of(schemaName)); Arrays.sort(nameIdentifiers1, Comparator.comparing(NameIdentifier::name)); Assertions.assertEquals(2, nameIdentifiers1.length, "should have 2 filesets"); Assertions.assertEquals(fileset1.name(), nameIdentifiers1[0].name()); @@ -432,8 +428,7 @@ public void testRenameFileset() throws IOException { catalog .asFilesetCatalog() .alterFileset( - NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName), - FilesetChange.rename(newFilesetName)); + NameIdentifier.of(schemaName, filesetName), FilesetChange.rename(newFilesetName)); // verify fileset is updated Assertions.assertNotNull(newFileset, "fileset should be created"); @@ -463,7 +458,7 @@ public void testFilesetUpdateComment() throws IOException { catalog .asFilesetCatalog() .alterFileset( - NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName), + NameIdentifier.of(schemaName, filesetName), FilesetChange.updateComment(newComment)); assertFilesetExists(filesetName); @@ -493,8 +488,7 @@ public void testFilesetSetProperties() throws IOException { catalog .asFilesetCatalog() .alterFileset( - NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName), - FilesetChange.setProperty("k1", "v2")); + NameIdentifier.of(schemaName, filesetName), FilesetChange.setProperty("k1", "v2")); assertFilesetExists(filesetName); // verify fileset is updated @@ -523,8 +517,7 @@ public void testFilesetRemoveProperties() throws IOException { catalog .asFilesetCatalog() .alterFileset( - NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName), - FilesetChange.removeProperty("k1")); + NameIdentifier.of(schemaName, filesetName), FilesetChange.removeProperty("k1")); assertFilesetExists(filesetName); // verify fileset is updated @@ -580,18 +573,12 @@ private Fileset createFileset( return catalog .asFilesetCatalog() .createFileset( - NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName), - comment, - type, - storageLocation, - properties); + NameIdentifier.of(schemaName, filesetName), comment, type, storageLocation, properties); } private void assertFilesetExists(String filesetName) throws IOException { Assertions.assertTrue( - catalog - .asFilesetCatalog() - .filesetExists(NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName)), + catalog.asFilesetCatalog().filesetExists(NameIdentifier.of(schemaName, filesetName)), "fileset should be exists"); Assertions.assertTrue( hdfs.exists(new Path(storageLocation(filesetName))), "storage location should be exists"); diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java index 030283c34d9..1e866ccaf66 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java @@ -1,14 +1,28 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop.integration.test; -import static com.datastrato.gravitino.catalog.hadoop.kerberos.AuthenticationConfig.AUTH_TYPE_KEY; -import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.IMPERSONATION_ENABLE_KEY; -import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.KEY_TAB_URI_KEY; -import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.PRINCIPAL_KEY; +import static com.datastrato.gravitino.catalog.hadoop.authentication.AuthenticationConfig.AUTH_TYPE_KEY; +import static com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.IMPERSONATION_ENABLE_KEY; +import static com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.KEY_TAB_URI_KEY; +import static com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.PRINCIPAL_KEY; import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.NameIdentifier; @@ -40,7 +54,7 @@ import org.slf4j.LoggerFactory; import sun.security.krb5.KrbException; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class HadoopUserAuthenticationIT extends AbstractIT { private static final Logger LOG = LoggerFactory.getLogger(HadoopUserAuthenticationIT.class); @@ -197,15 +211,13 @@ public void testUserAuthentication() { catalog .asFilesetCatalog() .createFileset( - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, SCHEMA_NAME, TABLE_NAME), + NameIdentifier.of(SCHEMA_NAME, TABLE_NAME), "comment", Fileset.Type.MANAGED, null, ImmutableMap.of()); - catalog - .asFilesetCatalog() - .dropFileset(NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, SCHEMA_NAME, TABLE_NAME)); + catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(SCHEMA_NAME, TABLE_NAME)); catalog.asSchemas().alterSchema(SCHEMA_NAME, SchemaChange.setProperty("k1", "value1")); diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java index ead0eb57ab8..b7f44350885 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java @@ -1,14 +1,28 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hadoop.integration.test; -import static com.datastrato.gravitino.catalog.hadoop.kerberos.AuthenticationConfig.AUTH_TYPE_KEY; -import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.IMPERSONATION_ENABLE_KEY; -import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.KEY_TAB_URI_KEY; -import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.PRINCIPAL_KEY; +import static com.datastrato.gravitino.catalog.hadoop.authentication.AuthenticationConfig.AUTH_TYPE_KEY; +import static com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.IMPERSONATION_ENABLE_KEY; +import static com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.KEY_TAB_URI_KEY; +import static com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.PRINCIPAL_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY; import com.datastrato.gravitino.Catalog; @@ -45,11 +59,13 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@Tag("gravitino-docker-test") public class HadoopUserImpersonationIT extends AbstractIT { private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogIT.class); @@ -291,11 +307,7 @@ private Fileset createFileset( return catalog .asFilesetCatalog() .createFileset( - NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName), - comment, - type, - storageLocation, - properties); + NameIdentifier.of(schemaName, filesetName), comment, type, storageLocation, properties); } private boolean checkFilePathExists(String pathString) throws Exception { diff --git a/catalogs/catalog-hadoop/src/test/resources/log4j2.properties b/catalogs/catalog-hadoop/src/test/resources/log4j2.properties index 8438fbd3d33..52f183579a4 100644 --- a/catalogs/catalog-hadoop/src/test/resources/log4j2.properties +++ b/catalogs/catalog-hadoop/src/test/resources/log4j2.properties @@ -1,6 +1,20 @@ # -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # Set to debug or trace if log4j initialization is failing diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index 65a36cd6b37..d0cad39098f 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ description = "catalog-hive" diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/FetchFileUtils.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/FetchFileUtils.java index 60505a7a340..80813a82fbe 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/FetchFileUtils.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/FetchFileUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalog.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalog.java index d92b5ee7e11..3963089896e 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalog.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java index 8e1eb188b6b..b42a7478a9e 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java index c9cbb6d19bf..165ab896d16 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java @@ -1,16 +1,34 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS; import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.CLIENT_POOL_SIZE; +import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.LIST_ALL_TABLES; import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.METASTORE_URIS; import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.PRINCIPAL; +import static com.datastrato.gravitino.catalog.hive.HiveTable.ICEBERG_TABLE_TYPE_VALUE; import static com.datastrato.gravitino.catalog.hive.HiveTable.SUPPORT_TABLE_TYPES; +import static com.datastrato.gravitino.catalog.hive.HiveTable.TABLE_TYPE_PROP; import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.COMMENT; import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TABLE_TYPE; +import static com.datastrato.gravitino.catalog.hive.converter.HiveDataTypeConverter.CONVERTER; import static com.datastrato.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; import static org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE; @@ -18,7 +36,6 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.SchemaChange; import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType; -import com.datastrato.gravitino.catalog.hive.converter.ToHiveType; import com.datastrato.gravitino.connector.CatalogInfo; import com.datastrato.gravitino.connector.CatalogOperations; import com.datastrato.gravitino.connector.HasPropertyMetadata; @@ -99,6 +116,7 @@ public class HiveCatalogOperations implements CatalogOperations, SupportsSchemas private ScheduledThreadPoolExecutor checkTgtExecutor; private String kerberosRealm; private ProxyPlugin proxyPlugin; + boolean listAllTables = true; // Map that maintains the mapping of keys in Gravitino to that in Hive, for example, users // will only need to set the configuration 'METASTORE_URL' in Gravitino and Gravitino will change @@ -150,6 +168,8 @@ public void initialize( this.clientPool = new CachedClientPool(getClientPoolSize(conf), hiveConf, getCacheEvictionInterval(conf)); + + this.listAllTables = enableListAllTables(conf); } private void initKerberosIfNecessary(Map conf, Configuration hadoopConf) { @@ -275,6 +295,10 @@ long getCacheEvictionInterval(Map conf) { .getOrDefault(conf, CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS); } + boolean enableListAllTables(Map conf) { + return (boolean) + propertiesMetadata.catalogPropertiesMetadata().getOrDefault(conf, LIST_ALL_TABLES); + } /** Closes the Hive catalog and releases the associated client pool. */ @Override public void close() { @@ -534,7 +558,18 @@ public NameIdentifier[] listTables(Namespace namespace) throws NoSuchSchemaExcep return clientPool.run( c -> c.getTableObjectsByName(schemaIdent.name(), allTables).stream() - .filter(tb -> SUPPORT_TABLE_TYPES.contains(tb.getTableType())) + .filter( + tb -> { + boolean isSupportTable = SUPPORT_TABLE_TYPES.contains(tb.getTableType()); + if (!isSupportTable) { + return false; + } + if (!listAllTables) { + Map parameters = tb.getParameters(); + return isHiveTable(parameters); + } + return true; + }) .map(tb -> NameIdentifier.of(namespace, tb.getTableName())) .toArray(NameIdentifier[]::new)); } catch (UnknownDBException e) { @@ -550,6 +585,22 @@ public NameIdentifier[] listTables(Namespace namespace) throws NoSuchSchemaExcep } } + boolean isHiveTable(Map tableParameters) { + if (isIcebergTable(tableParameters)) return false; + return true; + } + + boolean isIcebergTable(Map tableParameters) { + if (tableParameters != null) { + boolean isIcebergTable = + ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(tableParameters.get(TABLE_TYPE_PROP)); + if (isIcebergTable) { + return true; + } + } + return false; + } + /** * Loads a table from the Hive Metastore. * @@ -946,7 +997,7 @@ private void doAddColumn(List cols, TableChange.AddColumn change) { targetPosition, new FieldSchema( change.fieldName()[0], - ToHiveType.convert(change.getDataType()).getQualifiedName(), + CONVERTER.fromGravitino(change.getDataType()).getQualifiedName(), change.getComment())); } @@ -994,7 +1045,8 @@ private void doUpdateColumnType(List cols, TableChange.UpdateColumn if (indexOfColumn == -1) { throw new IllegalArgumentException("UpdateColumnType does not exist: " + columnName); } - cols.get(indexOfColumn).setType(ToHiveType.convert(change.getNewDataType()).getQualifiedName()); + cols.get(indexOfColumn) + .setType(CONVERTER.fromGravitino(change.getNewDataType()).getQualifiedName()); } /** diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogPropertiesMeta.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogPropertiesMeta.java index 29cb01b1266..75575ee2cc2 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogPropertiesMeta.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogPropertiesMeta.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; @@ -36,6 +50,10 @@ public class HiveCatalogPropertiesMeta extends BaseCatalogPropertiesMetadata { public static final String FETCH_TIMEOUT_SEC = "kerberos.keytab-fetch-timeout-sec"; + public static final String LIST_ALL_TABLES = "list-all-tables"; + + public static final boolean DEFAULT_LIST_ALL_TABLES = false; + private static final Map> HIVE_CATALOG_PROPERTY_ENTRIES = ImmutableMap.>builder() .put( @@ -88,6 +106,16 @@ public class HiveCatalogPropertiesMeta extends BaseCatalogPropertiesMetadata { FETCH_TIMEOUT_SEC, PropertyEntry.integerOptionalPropertyEntry( FETCH_TIMEOUT_SEC, "The timeout to fetch key tab", true, 60, false)) + .put( + LIST_ALL_TABLES, + PropertyEntry.booleanPropertyEntry( + LIST_ALL_TABLES, + "Lists all tables in a database, including non-Hive tables, such as Iceberg, etc.", + false, + false, + DEFAULT_LIST_ALL_TABLES, + false, + false)) .putAll(BASIC_CATALOG_PROPERTY_ENTRIES) .build(); diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveColumn.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveColumn.java index dfeeb618d30..1fc1d4fea12 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveColumn.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveColumn.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveProxyPlugin.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveProxyPlugin.java index 1b46d60fcf3..8d10a324db2 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveProxyPlugin.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveProxyPlugin.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveSchema.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveSchema.java index 954fbb1b7d4..11c5c30afcd 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveSchema.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveSchema.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveSchemaPropertiesMetadata.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveSchemaPropertiesMetadata.java index 8a1391df1fd..dc04f666601 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveSchemaPropertiesMetadata.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveSchemaPropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java index f33ec12d4b0..9b812b33a89 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; @@ -16,11 +30,10 @@ import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TABLE_TYPE; import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType.EXTERNAL_TABLE; import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType.MANAGED_TABLE; +import static com.datastrato.gravitino.catalog.hive.converter.HiveDataTypeConverter.CONVERTER; import static com.datastrato.gravitino.rel.expressions.transforms.Transforms.identity; import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType; -import com.datastrato.gravitino.catalog.hive.converter.FromHiveType; -import com.datastrato.gravitino.catalog.hive.converter.ToHiveType; import com.datastrato.gravitino.connector.BaseTable; import com.datastrato.gravitino.connector.PropertiesMetadata; import com.datastrato.gravitino.connector.TableOperations; @@ -62,6 +75,8 @@ public class HiveTable extends BaseTable { // A set of supported Hive table types. public static final Set SUPPORT_TABLE_TYPES = Sets.newHashSet(MANAGED_TABLE.name(), EXTERNAL_TABLE.name()); + public static final String ICEBERG_TABLE_TYPE_VALUE = "ICEBERG"; + public static final String TABLE_TYPE_PROP = "table_type"; private String schemaName; private CachedClientPool clientPool; private StorageDescriptor sd; @@ -112,7 +127,7 @@ public static HiveTable.Builder fromHiveTable(Table table) { f -> HiveColumn.builder() .withName(f.getName()) - .withType(FromHiveType.convert(f.getType())) + .withType(CONVERTER.toGravitino(f.getType())) .withComment(f.getComment()) .build()), table.getPartitionKeys().stream() @@ -120,7 +135,7 @@ public static HiveTable.Builder fromHiveTable(Table table) { p -> HiveColumn.builder() .withName(p.getName()) - .withType(FromHiveType.convert(p.getType())) + .withType(CONVERTER.toGravitino(p.getType())) .withComment(p.getComment()) .build())) .toArray(Column[]::new); @@ -239,7 +254,7 @@ private FieldSchema getPartitionKey(String[] fieldName) { .collect(Collectors.toList()); return new FieldSchema( partitionColumns.get(0).name(), - ToHiveType.convert(partitionColumns.get(0).dataType()).getQualifiedName(), + CONVERTER.fromGravitino(partitionColumns.get(0).dataType()).getQualifiedName(), partitionColumns.get(0).comment()); } @@ -254,7 +269,9 @@ private StorageDescriptor buildStorageDescriptor( .map( c -> new FieldSchema( - c.name(), ToHiveType.convert(c.dataType()).getQualifiedName(), c.comment())) + c.name(), + CONVERTER.fromGravitino(c.dataType()).getQualifiedName(), + c.comment())) .collect(Collectors.toList())); // `location` must not be null, otherwise it will result in an NPE when calling HMS `alterTable` diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java index 97ebbbf909f..572c8384ebe 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java @@ -1,10 +1,24 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; -import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.MetadataObjects; import com.datastrato.gravitino.connector.TableOperations; import com.datastrato.gravitino.exceptions.NoSuchPartitionException; import com.datastrato.gravitino.exceptions.NoSuchTableException; @@ -133,7 +147,7 @@ private String[][] getFieldNames(String partitionName) { @Override public Partition addPartition(Partition partition) throws PartitionAlreadyExistsException { - if (Entity.SECURABLE_ENTITY_RESERVED_NAME.equals(partition.name())) { + if (MetadataObjects.METADATA_OBJECT_RESERVED_NAME.equals(partition.name())) { throw new IllegalArgumentException("Can't create a catalog with with reserved partition `*`"); } diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTablePropertiesMetadata.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTablePropertiesMetadata.java index 0be2271a1de..373dc36ee50 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTablePropertiesMetadata.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTablePropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/FromHiveType.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/FromHiveType.java deleted file mode 100644 index 6d3e6e3477e..00000000000 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/FromHiveType.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.catalog.hive.converter; - -import static org.apache.hadoop.hive.serde.serdeConstants.BIGINT_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.BINARY_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.BOOLEAN_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.DATE_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.DOUBLE_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.FLOAT_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.INT_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.SMALLINT_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.TIMESTAMP_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.TINYINT_TYPE_NAME; -import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getTypeInfoFromTypeString; - -import com.datastrato.gravitino.rel.types.Type; -import com.datastrato.gravitino.rel.types.Types; -import com.google.common.annotations.VisibleForTesting; -import java.util.ArrayList; -import java.util.stream.IntStream; -import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; - -/** Converts Hive data types to corresponding Gravitino data types. */ -public class FromHiveType { - - /** - * Converts a Hive data type string to the corresponding Gravitino data type. - * - * @param hiveType The Hive data type string to convert. - * @return The equivalent Gravitino data type. - */ - public static Type convert(String hiveType) { - TypeInfo hiveTypeInfo = getTypeInfoFromTypeString(hiveType); - return toGravitinoType(hiveTypeInfo); - } - - /** - * Converts a Hive TypeInfo object to the corresponding Gravitino Type. - * - * @param hiveTypeInfo The Hive TypeInfo object to convert. - * @return The equivalent Gravitino Type. - */ - @VisibleForTesting - public static Type toGravitinoType(TypeInfo hiveTypeInfo) { - switch (hiveTypeInfo.getCategory()) { - case PRIMITIVE: - switch (hiveTypeInfo.getTypeName()) { - case BOOLEAN_TYPE_NAME: - return Types.BooleanType.get(); - case TINYINT_TYPE_NAME: - return Types.ByteType.get(); - case SMALLINT_TYPE_NAME: - return Types.ShortType.get(); - case INT_TYPE_NAME: - return Types.IntegerType.get(); - case BIGINT_TYPE_NAME: - return Types.LongType.get(); - case FLOAT_TYPE_NAME: - return Types.FloatType.get(); - case DOUBLE_TYPE_NAME: - return Types.DoubleType.get(); - case STRING_TYPE_NAME: - return Types.StringType.get(); - case DATE_TYPE_NAME: - return Types.DateType.get(); - case TIMESTAMP_TYPE_NAME: - return Types.TimestampType.withoutTimeZone(); - case BINARY_TYPE_NAME: - return Types.BinaryType.get(); - case INTERVAL_YEAR_MONTH_TYPE_NAME: - return Types.IntervalYearType.get(); - case INTERVAL_DAY_TIME_TYPE_NAME: - return Types.IntervalDayType.get(); - default: - if (hiveTypeInfo instanceof CharTypeInfo) { - return Types.FixedCharType.of(((CharTypeInfo) hiveTypeInfo).getLength()); - } - - if (hiveTypeInfo instanceof VarcharTypeInfo) { - return Types.VarCharType.of(((VarcharTypeInfo) hiveTypeInfo).getLength()); - } - - if (hiveTypeInfo instanceof DecimalTypeInfo) { - DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) hiveTypeInfo; - return Types.DecimalType.of(decimalTypeInfo.precision(), decimalTypeInfo.scale()); - } - - return Types.ExternalType.of(hiveTypeInfo.getQualifiedName()); - } - case LIST: - return Types.ListType.nullable( - toGravitinoType(((ListTypeInfo) hiveTypeInfo).getListElementTypeInfo())); - case MAP: - MapTypeInfo mapTypeInfo = (MapTypeInfo) hiveTypeInfo; - return Types.MapType.valueNullable( - toGravitinoType(mapTypeInfo.getMapKeyTypeInfo()), - toGravitinoType(mapTypeInfo.getMapValueTypeInfo())); - case STRUCT: - StructTypeInfo structTypeInfo = (StructTypeInfo) hiveTypeInfo; - ArrayList fieldNames = structTypeInfo.getAllStructFieldNames(); - ArrayList typeInfos = structTypeInfo.getAllStructFieldTypeInfos(); - Types.StructType.Field[] fields = - IntStream.range(0, fieldNames.size()) - .mapToObj( - i -> - Types.StructType.Field.nullableField( - fieldNames.get(i), toGravitinoType(typeInfos.get(i)))) - .toArray(Types.StructType.Field[]::new); - return Types.StructType.of(fields); - case UNION: - UnionTypeInfo unionTypeInfo = (UnionTypeInfo) hiveTypeInfo; - return Types.UnionType.of( - unionTypeInfo.getAllUnionObjectTypeInfos().stream() - .map(FromHiveType::toGravitinoType) - .toArray(Type[]::new)); - default: - return Types.ExternalType.of(hiveTypeInfo.getQualifiedName()); - } - } - - private FromHiveType() {} -} diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/HiveDataTypeConverter.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/HiveDataTypeConverter.java new file mode 100644 index 00000000000..e4a53760bb1 --- /dev/null +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/HiveDataTypeConverter.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.hive.converter; + +import static org.apache.hadoop.hive.serde.serdeConstants.BIGINT_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.BINARY_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.BOOLEAN_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.DATE_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.DOUBLE_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.FLOAT_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.INT_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.SMALLINT_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.TIMESTAMP_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.TINYINT_TYPE_NAME; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getCharTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getDecimalTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getListTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getMapTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getPrimitiveTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getStructTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getUnionTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getVarcharTypeInfo; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getTypeInfoFromTypeString; + +import com.datastrato.gravitino.connector.DataTypeConverter; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; + +public class HiveDataTypeConverter implements DataTypeConverter { + public static final HiveDataTypeConverter CONVERTER = new HiveDataTypeConverter(); + + @Override + public TypeInfo fromGravitino(Type type) { + switch (type.name()) { + case BOOLEAN: + return getPrimitiveTypeInfo(BOOLEAN_TYPE_NAME); + case BYTE: + return getPrimitiveTypeInfo(TINYINT_TYPE_NAME); + case SHORT: + return getPrimitiveTypeInfo(SMALLINT_TYPE_NAME); + case INTEGER: + return getPrimitiveTypeInfo(INT_TYPE_NAME); + case LONG: + return getPrimitiveTypeInfo(BIGINT_TYPE_NAME); + case FLOAT: + return getPrimitiveTypeInfo(FLOAT_TYPE_NAME); + case DOUBLE: + return getPrimitiveTypeInfo(DOUBLE_TYPE_NAME); + case STRING: + return getPrimitiveTypeInfo(STRING_TYPE_NAME); + case VARCHAR: + return getVarcharTypeInfo(((Types.VarCharType) type).length()); + case FIXEDCHAR: + return getCharTypeInfo(((Types.FixedCharType) type).length()); + case DATE: + return getPrimitiveTypeInfo(DATE_TYPE_NAME); + case TIMESTAMP: + return getPrimitiveTypeInfo(TIMESTAMP_TYPE_NAME); + case DECIMAL: + Types.DecimalType decimalType = (Types.DecimalType) type; + return getDecimalTypeInfo(decimalType.precision(), decimalType.scale()); + case BINARY: + return getPrimitiveTypeInfo(BINARY_TYPE_NAME); + case INTERVAL_YEAR: + return getPrimitiveTypeInfo(INTERVAL_YEAR_MONTH_TYPE_NAME); + case INTERVAL_DAY: + return getPrimitiveTypeInfo(INTERVAL_DAY_TIME_TYPE_NAME); + case LIST: + return getListTypeInfo(fromGravitino(((Types.ListType) type).elementType())); + case MAP: + Types.MapType mapType = (Types.MapType) type; + return getMapTypeInfo(fromGravitino(mapType.keyType()), fromGravitino(mapType.valueType())); + case STRUCT: + Types.StructType structType = (Types.StructType) type; + List typeInfos = + Arrays.stream(structType.fields()) + .map(t -> fromGravitino(t.type())) + .collect(Collectors.toList()); + List names = + Arrays.stream(structType.fields()) + .map(Types.StructType.Field::name) + .collect(Collectors.toList()); + return getStructTypeInfo(names, typeInfos); + case UNION: + return getUnionTypeInfo( + Arrays.stream(((Types.UnionType) type).types()) + .map(this::fromGravitino) + .collect(Collectors.toList())); + default: + throw new UnsupportedOperationException("Unsupported conversion to Hive type: " + type); + } + } + + @Override + public Type toGravitino(String hiveType) { + return toGravitino(getTypeInfoFromTypeString(hiveType)); + } + + private Type toGravitino(TypeInfo hiveTypeInfo) { + switch (hiveTypeInfo.getCategory()) { + case PRIMITIVE: + switch (hiveTypeInfo.getTypeName()) { + case BOOLEAN_TYPE_NAME: + return Types.BooleanType.get(); + case TINYINT_TYPE_NAME: + return Types.ByteType.get(); + case SMALLINT_TYPE_NAME: + return Types.ShortType.get(); + case INT_TYPE_NAME: + return Types.IntegerType.get(); + case BIGINT_TYPE_NAME: + return Types.LongType.get(); + case FLOAT_TYPE_NAME: + return Types.FloatType.get(); + case DOUBLE_TYPE_NAME: + return Types.DoubleType.get(); + case STRING_TYPE_NAME: + return Types.StringType.get(); + case DATE_TYPE_NAME: + return Types.DateType.get(); + case TIMESTAMP_TYPE_NAME: + return Types.TimestampType.withoutTimeZone(); + case BINARY_TYPE_NAME: + return Types.BinaryType.get(); + case INTERVAL_YEAR_MONTH_TYPE_NAME: + return Types.IntervalYearType.get(); + case INTERVAL_DAY_TIME_TYPE_NAME: + return Types.IntervalDayType.get(); + default: + if (hiveTypeInfo instanceof CharTypeInfo) { + return Types.FixedCharType.of(((CharTypeInfo) hiveTypeInfo).getLength()); + } + + if (hiveTypeInfo instanceof VarcharTypeInfo) { + return Types.VarCharType.of(((VarcharTypeInfo) hiveTypeInfo).getLength()); + } + + if (hiveTypeInfo instanceof DecimalTypeInfo) { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) hiveTypeInfo; + return Types.DecimalType.of(decimalTypeInfo.precision(), decimalTypeInfo.scale()); + } + + return Types.ExternalType.of(hiveTypeInfo.getQualifiedName()); + } + case LIST: + return Types.ListType.nullable( + toGravitino(((ListTypeInfo) hiveTypeInfo).getListElementTypeInfo())); + case MAP: + MapTypeInfo mapTypeInfo = (MapTypeInfo) hiveTypeInfo; + return Types.MapType.valueNullable( + toGravitino(mapTypeInfo.getMapKeyTypeInfo()), + toGravitino(mapTypeInfo.getMapValueTypeInfo())); + case STRUCT: + StructTypeInfo structTypeInfo = (StructTypeInfo) hiveTypeInfo; + ArrayList fieldNames = structTypeInfo.getAllStructFieldNames(); + ArrayList typeInfos = structTypeInfo.getAllStructFieldTypeInfos(); + Types.StructType.Field[] fields = + IntStream.range(0, fieldNames.size()) + .mapToObj( + i -> + Types.StructType.Field.nullableField( + fieldNames.get(i), toGravitino(typeInfos.get(i)))) + .toArray(Types.StructType.Field[]::new); + return Types.StructType.of(fields); + case UNION: + UnionTypeInfo unionTypeInfo = (UnionTypeInfo) hiveTypeInfo; + return Types.UnionType.of( + unionTypeInfo.getAllUnionObjectTypeInfos().stream() + .map(this::toGravitino) + .toArray(Type[]::new)); + default: + return Types.ExternalType.of(hiveTypeInfo.getQualifiedName()); + } + } +} diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/ToHiveType.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/ToHiveType.java deleted file mode 100644 index b29481ebc4c..00000000000 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/ToHiveType.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.catalog.hive.converter; - -import static org.apache.hadoop.hive.serde.serdeConstants.BIGINT_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.BINARY_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.BOOLEAN_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.DATE_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.DOUBLE_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.FLOAT_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.INT_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.SMALLINT_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.TIMESTAMP_TYPE_NAME; -import static org.apache.hadoop.hive.serde.serdeConstants.TINYINT_TYPE_NAME; -import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getCharTypeInfo; -import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getDecimalTypeInfo; -import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getListTypeInfo; -import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getMapTypeInfo; -import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getPrimitiveTypeInfo; -import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getStructTypeInfo; -import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getUnionTypeInfo; -import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getVarcharTypeInfo; - -import com.datastrato.gravitino.rel.types.Type; -import com.datastrato.gravitino.rel.types.Types; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; - -/** Converts Gravitino data types to corresponding Hive data types. */ -public class ToHiveType { - public static TypeInfo convert(Type type) { - switch (type.name()) { - case BOOLEAN: - return getPrimitiveTypeInfo(BOOLEAN_TYPE_NAME); - case BYTE: - return getPrimitiveTypeInfo(TINYINT_TYPE_NAME); - case SHORT: - return getPrimitiveTypeInfo(SMALLINT_TYPE_NAME); - case INTEGER: - return getPrimitiveTypeInfo(INT_TYPE_NAME); - case LONG: - return getPrimitiveTypeInfo(BIGINT_TYPE_NAME); - case FLOAT: - return getPrimitiveTypeInfo(FLOAT_TYPE_NAME); - case DOUBLE: - return getPrimitiveTypeInfo(DOUBLE_TYPE_NAME); - case STRING: - return getPrimitiveTypeInfo(STRING_TYPE_NAME); - case VARCHAR: - return getVarcharTypeInfo(((Types.VarCharType) type).length()); - case FIXEDCHAR: - return getCharTypeInfo(((Types.FixedCharType) type).length()); - case DATE: - return getPrimitiveTypeInfo(DATE_TYPE_NAME); - case TIMESTAMP: - return getPrimitiveTypeInfo(TIMESTAMP_TYPE_NAME); - case DECIMAL: - Types.DecimalType decimalType = (Types.DecimalType) type; - return getDecimalTypeInfo(decimalType.precision(), decimalType.scale()); - case BINARY: - return getPrimitiveTypeInfo(BINARY_TYPE_NAME); - case INTERVAL_YEAR: - return getPrimitiveTypeInfo(INTERVAL_YEAR_MONTH_TYPE_NAME); - case INTERVAL_DAY: - return getPrimitiveTypeInfo(INTERVAL_DAY_TIME_TYPE_NAME); - case LIST: - return getListTypeInfo(convert(((Types.ListType) type).elementType())); - case MAP: - Types.MapType mapType = (Types.MapType) type; - return getMapTypeInfo(convert(mapType.keyType()), convert(mapType.valueType())); - case STRUCT: - Types.StructType structType = (Types.StructType) type; - List typeInfos = - Arrays.stream(structType.fields()) - .map(t -> convert(t.type())) - .collect(Collectors.toList()); - List names = - Arrays.stream(structType.fields()) - .map(Types.StructType.Field::name) - .collect(Collectors.toList()); - return getStructTypeInfo(names, typeInfos); - case UNION: - return getUnionTypeInfo( - Arrays.stream(((Types.UnionType) type).types()) - .map(ToHiveType::convert) - .collect(Collectors.toList())); - default: - throw new UnsupportedOperationException("Unsupported conversion to Hive type: " + type); - } - } - - private ToHiveType() {} -} diff --git a/catalogs/catalog-hive/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider b/catalogs/catalog-hive/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider index c14b32714a9..060ce5dc269 100644 --- a/catalogs/catalog-hive/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider +++ b/catalogs/catalog-hive/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider @@ -1,5 +1,19 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # com.datastrato.gravitino.catalog.hive.HiveCatalog \ No newline at end of file diff --git a/catalogs/catalog-hive/src/main/resources/hive-site.xml.template b/catalogs/catalog-hive/src/main/resources/hive-site.xml.template index 263112f905f..efa7db5adb6 100644 --- a/catalogs/catalog-hive/src/main/resources/hive-site.xml.template +++ b/catalogs/catalog-hive/src/main/resources/hive-site.xml.template @@ -1,6 +1,20 @@ diff --git a/catalogs/catalog-hive/src/main/resources/hive.conf b/catalogs/catalog-hive/src/main/resources/hive.conf index c2276a13ee9..b7e07493ff4 100644 --- a/catalogs/catalog-hive/src/main/resources/hive.conf +++ b/catalogs/catalog-hive/src/main/resources/hive.conf @@ -1,6 +1,20 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # ## This file holds common properties for hive catalog and metastore, for more, please refer to diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestFetchFileUtils.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestFetchFileUtils.java index 3b89c89fe62..d0b2dfb2123 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestFetchFileUtils.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestFetchFileUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalog.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalog.java index ac84af7dee1..96b6dcecea8 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalog.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalogOperations.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalogOperations.java index 27aae03327a..edf23bab8ca 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalogOperations.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalogOperations.java @@ -1,16 +1,33 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; +import static com.datastrato.gravitino.Catalog.CLOUD_NAME; +import static com.datastrato.gravitino.Catalog.CLOUD_REGION_CODE; import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.CHECK_INTERVAL_SEC; import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS; import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.CLIENT_POOL_SIZE; import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.FETCH_TIMEOUT_SEC; import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.IMPERSONATION_ENABLE; import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.KEY_TAB_URI; +import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.LIST_ALL_TABLES; import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.METASTORE_URIS; import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.PRINCIPAL; import static com.datastrato.gravitino.catalog.hive.TestHiveCatalog.HIVE_PROPERTIES_METADATA; @@ -67,12 +84,13 @@ void testPropertyMeta() { Map> propertyEntryMap = HIVE_PROPERTIES_METADATA.catalogPropertiesMetadata().propertyEntries(); - Assertions.assertEquals(11, propertyEntryMap.size()); + Assertions.assertEquals(14, propertyEntryMap.size()); Assertions.assertTrue(propertyEntryMap.containsKey(METASTORE_URIS)); Assertions.assertTrue(propertyEntryMap.containsKey(Catalog.PROPERTY_PACKAGE)); Assertions.assertTrue(propertyEntryMap.containsKey(BaseCatalog.CATALOG_OPERATION_IMPL)); Assertions.assertTrue(propertyEntryMap.containsKey(CLIENT_POOL_SIZE)); Assertions.assertTrue(propertyEntryMap.containsKey(IMPERSONATION_ENABLE)); + Assertions.assertTrue(propertyEntryMap.containsKey(LIST_ALL_TABLES)); Assertions.assertTrue(propertyEntryMap.get(METASTORE_URIS).isRequired()); Assertions.assertFalse(propertyEntryMap.get(Catalog.PROPERTY_PACKAGE).isRequired()); @@ -84,6 +102,8 @@ void testPropertyMeta() { Assertions.assertFalse(propertyEntryMap.get(PRINCIPAL).isRequired()); Assertions.assertFalse(propertyEntryMap.get(CHECK_INTERVAL_SEC).isRequired()); Assertions.assertFalse(propertyEntryMap.get(FETCH_TIMEOUT_SEC).isRequired()); + Assertions.assertFalse(propertyEntryMap.get(CLOUD_NAME).isRequired()); + Assertions.assertFalse(propertyEntryMap.get(CLOUD_REGION_CODE).isRequired()); } @Test diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveSchema.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveSchema.java index db067fc965a..71c96252a64 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveSchema.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveSchema.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java index c4b9306db65..63ffd452b80 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTable.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTableOperations.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTableOperations.java index 501095a0ef4..c07ce077f7a 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTableOperations.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive; diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/converter/TestTypeConverter.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/converter/TestTypeConverter.java index 15947229893..b11a510a556 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/converter/TestTypeConverter.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/converter/TestTypeConverter.java @@ -1,9 +1,24 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive.converter; +import static com.datastrato.gravitino.catalog.hive.converter.HiveDataTypeConverter.CONVERTER; import static org.apache.hadoop.hive.serde.serdeConstants.BIGINT_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.BINARY_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.BOOLEAN_TYPE_NAME; @@ -29,7 +44,6 @@ import com.datastrato.gravitino.rel.types.Types; import java.util.Arrays; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -72,24 +86,14 @@ public void testTypeConverter() { Arrays.asList( getPrimitiveTypeInfo(STRING_TYPE_NAME), getPrimitiveTypeInfo(INT_TYPE_NAME))) .getTypeName()); - Assertions.assertEquals( - Types.ExternalType.of(USER_DEFINED_TYPE), - FromHiveType.toGravitinoType(new UserDefinedTypeInfo())); Assertions.assertThrows( UnsupportedOperationException.class, - () -> ToHiveType.convert(Types.ExternalType.of(USER_DEFINED_TYPE))); + () -> CONVERTER.fromGravitino(Types.ExternalType.of(USER_DEFINED_TYPE))); } private void testConverter(String typeName) { TypeInfo hiveType = getTypeInfoFromTypeString(typeName); - TypeInfo convertedType = ToHiveType.convert(FromHiveType.convert(hiveType.getTypeName())); + TypeInfo convertedType = CONVERTER.fromGravitino(CONVERTER.toGravitino(hiveType.getTypeName())); Assertions.assertEquals(hiveType, convertedType); } - - static class UserDefinedTypeInfo extends PrimitiveTypeInfo { - @Override - public String getTypeName() { - return USER_DEFINED_TYPE; - } - } } diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/dyn/TestDynFields.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/dyn/TestDynFields.java index 2d350bb0f02..07d5036c187 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/dyn/TestDynFields.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/dyn/TestDynFields.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive.dyn; diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java index f6fcb1e971b..6e5853a27e3 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive.integration.test; @@ -101,7 +115,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class CatalogHiveIT extends AbstractIT { private static final Logger LOG = LoggerFactory.getLogger(CatalogHiveIT.class); public static final String metalakeName = @@ -199,18 +213,18 @@ public static void startup() throws Exception { @AfterAll public static void stop() throws IOException { - Arrays.stream(catalog.asSchemas().listSchemas()) - .filter(schema -> !schema.equals("default")) - .forEach( - (schema -> { - catalog.asSchemas().dropSchema(schema, true); - })); - Arrays.stream(metalake.listCatalogs()) - .forEach( - (ident -> { - metalake.dropCatalog(ident.name()); - })); if (client != null) { + Arrays.stream(catalog.asSchemas().listSchemas()) + .filter(schema -> !schema.equals("default")) + .forEach( + (schema -> { + catalog.asSchemas().dropSchema(schema, true); + })); + Arrays.stream(metalake.listCatalogs()) + .forEach( + (catalogName -> { + metalake.dropCatalog(catalogName); + })); client.dropMetalake(metalakeName); } if (hiveClientPool != null) { @@ -230,7 +244,6 @@ public static void stop() throws IOException { LOG.error("Failed to close CloseableGroup", e); } - AbstractIT.customConfigs.clear(); AbstractIT.client = null; } @@ -265,7 +278,6 @@ private static void createCatalog() { } private static void createSchema() throws TException, InterruptedException { - NameIdentifier ident = NameIdentifier.of(metalakeName, catalogName, schemaName); Map properties = Maps.newHashMap(); properties.put("key1", "val1"); properties.put("key2", "val2"); @@ -278,8 +290,8 @@ private static void createSchema() throws TException, InterruptedException { schemaName.toLowerCase())); String comment = "comment"; - catalog.asSchemas().createSchema(ident.name(), comment, properties); - Schema loadSchema = catalog.asSchemas().loadSchema(ident.name()); + catalog.asSchemas().createSchema(schemaName, comment, properties); + Schema loadSchema = catalog.asSchemas().loadSchema(schemaName); Assertions.assertEquals(schemaName.toLowerCase(), loadSchema.name()); Assertions.assertEquals(comment, loadSchema.comment()); Assertions.assertEquals("val1", loadSchema.properties().get("key1")); @@ -349,8 +361,7 @@ public void testCreateHiveTableWithDistributionAndSortOrder() // Create table from Gravitino API Column[] columns = createColumns(); - NameIdentifier nameIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier nameIdentifier = NameIdentifier.of(schemaName, tableName); Distribution distribution = Distributions.of(Strategy.EVEN, 10, NamedReference.field(HIVE_COL_NAME1)); @@ -449,8 +460,7 @@ public void testCreateHiveTable() throws TException, InterruptedException { // Create table from Gravitino API Column[] columns = createColumns(); - NameIdentifier nameIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier nameIdentifier = NameIdentifier.of(schemaName, tableName); Map properties = createProperties(); Table createdTable = catalog @@ -547,8 +557,7 @@ public void testCreateHiveTable() throws TException, InterruptedException { @Test public void testHiveTableProperties() throws TException, InterruptedException { Column[] columns = createColumns(); - NameIdentifier nameIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier nameIdentifier = NameIdentifier.of(schemaName, tableName); // test default properties Table createdTable = catalog @@ -571,7 +580,7 @@ public void testHiveTableProperties() throws TException, InterruptedException { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, table2), + NameIdentifier.of(schemaName, table2), columns, TABLE_COMMENT, ImmutableMap.of( @@ -608,7 +617,7 @@ public void testHiveTableProperties() throws TException, InterruptedException { // test alter properties exception TableCatalog tableCatalog = catalog.asTableCatalog(); - NameIdentifier id = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier id = NameIdentifier.of(schemaName, tableName); TableChange change = TableChange.setProperty(TRANSIENT_LAST_DDL_TIME, "1234"); IllegalArgumentException exception = assertThrows( @@ -622,8 +631,7 @@ public void testHiveTableProperties() throws TException, InterruptedException { @Test public void testHiveSchemaProperties() throws TException, InterruptedException { // test LOCATION property - NameIdentifier schemaIdent = - NameIdentifier.of(metalakeName, catalogName, GravitinoITUtils.genRandomName(SCHEMA_PREFIX)); + String schemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX); Map properties = Maps.newHashMap(); String expectedSchemaLocation = String.format( @@ -632,18 +640,14 @@ public void testHiveSchemaProperties() throws TException, InterruptedException { HiveContainer.HDFS_DEFAULTFS_PORT); properties.put(HiveSchemaPropertiesMetadata.LOCATION, expectedSchemaLocation); - catalog.asSchemas().createSchema(schemaIdent.name(), "comment", properties); + catalog.asSchemas().createSchema(schemaName, "comment", properties); - Database actualSchema = hiveClientPool.run(client -> client.getDatabase(schemaIdent.name())); + Database actualSchema = hiveClientPool.run(client -> client.getDatabase(schemaName)); String actualSchemaLocation = actualSchema.getLocationUri(); Assertions.assertTrue(actualSchemaLocation.endsWith(expectedSchemaLocation)); NameIdentifier tableIdent = - NameIdentifier.of( - metalakeName, - catalogName, - schemaIdent.name(), - GravitinoITUtils.genRandomName(TABLE_PREFIX)); + NameIdentifier.of(schemaName, GravitinoITUtils.genRandomName(TABLE_PREFIX)); catalog .asTableCatalog() .createTable( @@ -653,7 +657,7 @@ public void testHiveSchemaProperties() throws TException, InterruptedException { ImmutableMap.of(), Transforms.EMPTY_TRANSFORM); org.apache.hadoop.hive.metastore.api.Table actualTable = - hiveClientPool.run(client -> client.getTable(schemaIdent.name(), tableIdent.name())); + hiveClientPool.run(client -> client.getTable(schemaName, tableIdent.name())); String actualTableLocation = actualTable.getSd().getLocation(); // use `tableIdent.name().toLowerCase()` because HMS will convert table name to lower String expectedTableLocation = expectedSchemaLocation + "/" + tableIdent.name().toLowerCase(); @@ -666,8 +670,7 @@ public void testCreatePartitionedHiveTable() throws TException, InterruptedExcep // Create table from Gravitino API Column[] columns = createColumns(); - NameIdentifier nameIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier nameIdentifier = NameIdentifier.of(schemaName, tableName); Map properties = createProperties(); Table createdTable = catalog @@ -714,8 +717,7 @@ public void testCreatePartitionedHiveTable() throws TException, InterruptedExcep public void testListPartitionNames() throws TException, InterruptedException { // test empty partitions Column[] columns = createColumns(); - NameIdentifier nameIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier nameIdentifier = NameIdentifier.of(schemaName, tableName); Table nonPartitionedTable = catalog .asTableCatalog() @@ -741,8 +743,7 @@ public void testListPartitionNames() throws TException, InterruptedException { public void testListPartitions() throws TException, InterruptedException { // test empty partitions Column[] columns = createColumns(); - NameIdentifier nameIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier nameIdentifier = NameIdentifier.of(schemaName, tableName); Table nonPartitionedTable = catalog .asTableCatalog() @@ -982,8 +983,7 @@ private Table preparePartitionedTable() throws TException, InterruptedException Column[] columns = createColumns(); NameIdentifier nameIdentifier = - NameIdentifier.of( - metalakeName, catalogName, schemaName, GravitinoITUtils.genRandomName(TABLE_PREFIX)); + NameIdentifier.of(schemaName, GravitinoITUtils.genRandomName(TABLE_PREFIX)); Map properties = createProperties(); Table table = catalog @@ -1059,7 +1059,7 @@ private void assertTableEquals( @Test void testAlterUnknownTable() { - NameIdentifier identifier = NameIdentifier.of(metalakeName, catalogName, schemaName, "unknown"); + NameIdentifier identifier = NameIdentifier.of(schemaName, "unknown"); TableCatalog tableCatalog = catalog.asTableCatalog(); TableChange change = TableChange.updateComment("new_comment"); Assertions.assertThrows( @@ -1076,7 +1076,7 @@ public void testAlterHiveTable() throws TException, InterruptedException { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), columns, TABLE_COMMENT, createProperties(), @@ -1087,7 +1087,7 @@ public void testAlterHiveTable() throws TException, InterruptedException { catalog .asTableCatalog() .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.rename(ALTER_TABLE_NAME), TableChange.updateComment(TABLE_COMMENT + "_new"), TableChange.removeProperty("key1"), @@ -1129,7 +1129,7 @@ public void testAlterHiveTable() throws TException, InterruptedException { // test alter partition column exception TableCatalog tableCatalog = catalog.asTableCatalog(); - NameIdentifier id = NameIdentifier.of(metalakeName, catalogName, schemaName, ALTER_TABLE_NAME); + NameIdentifier id = NameIdentifier.of(schemaName, ALTER_TABLE_NAME); TableChange updateType = TableChange.updateColumnType(new String[] {HIVE_COL_NAME3}, Types.IntegerType.get()); RuntimeException exception = @@ -1190,11 +1190,7 @@ public void testAlterHiveTable() throws TException, InterruptedException { Column[] newColumns = new Column[] {col1, col2, col3}; NameIdentifier tableIdentifier = - NameIdentifier.of( - metalakeName, - catalogName, - schemaName, - GravitinoITUtils.genRandomName("CatalogHiveIT_table")); + NameIdentifier.of(schemaName, GravitinoITUtils.genRandomName("CatalogHiveIT_table")); catalog .asTableCatalog() .createTable( @@ -1249,14 +1245,12 @@ public void testDropHiveTable() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), createColumns(), TABLE_COMMENT, createProperties(), Transforms.EMPTY_TRANSFORM); - catalog - .asTableCatalog() - .dropTable(NameIdentifier.of(metalakeName, catalogName, schemaName, ALTER_TABLE_NAME)); + catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, ALTER_TABLE_NAME)); // Directly get table from hive metastore to check if the table is dropped successfully. assertThrows( @@ -1266,25 +1260,23 @@ public void testDropHiveTable() { @Test public void testAlterSchema() throws TException, InterruptedException { - NameIdentifier ident = NameIdentifier.of(metalakeName, catalogName, schemaName); - GravitinoMetalake metalake = client.loadMetalake(metalakeName); Catalog catalog = metalake.loadCatalog(catalogName); - Schema schema = catalog.asSchemas().loadSchema(ident.name()); + Schema schema = catalog.asSchemas().loadSchema(schemaName); Assertions.assertNull(schema.auditInfo().lastModifier()); Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, schema.auditInfo().creator()); schema = catalog .asSchemas() .alterSchema( - ident.name(), + schemaName, SchemaChange.removeProperty("key1"), SchemaChange.setProperty("key2", "val2-alter")); Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, schema.auditInfo().lastModifier()); Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, schema.auditInfo().creator()); - Map properties2 = catalog.asSchemas().loadSchema(ident.name()).properties(); + Map properties2 = catalog.asSchemas().loadSchema(schemaName).properties(); Assertions.assertFalse(properties2.containsKey("key1")); Assertions.assertEquals("val2-alter", properties2.get("key2")); @@ -1325,12 +1317,11 @@ void testLoadEntityWithSamePrefix() { for (int i = 1; i < schemaName.length(); i++) { // We can't get the schema by prefix final int length = i; - final NameIdentifier id = - NameIdentifier.of(metalakeName, catalogName, schemaName.substring(0, length)); + final NameIdentifier id = NameIdentifier.of(schemaName.substring(0, length)); Assertions.assertThrows(NoSuchSchemaException.class, () -> schemas.loadSchema(id.name())); } - NameIdentifier idC = NameIdentifier.of(metalakeName, catalogName, schemaName + "a"); + NameIdentifier idC = NameIdentifier.of(schemaName + "a"); Assertions.assertThrows(NoSuchSchemaException.class, () -> schemas.loadSchema(idC.name())); TableCatalog tableCatalog = catalog.asTableCatalog(); @@ -1338,12 +1329,11 @@ void testLoadEntityWithSamePrefix() { for (int i = 1; i < tableName.length(); i++) { // We can't get the table by prefix final int length = i; - final NameIdentifier id = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName.substring(0, length)); + final NameIdentifier id = NameIdentifier.of(schemaName, tableName.substring(0, length)); Assertions.assertThrows(NoSuchTableException.class, () -> tableCatalog.loadTable(id)); } - NameIdentifier idD = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName + "a"); + NameIdentifier idD = NameIdentifier.of(schemaName, tableName + "a"); Assertions.assertThrows(NoSuchTableException.class, () -> tableCatalog.loadTable(idD)); } @@ -1409,14 +1399,14 @@ void testAlterEntityName() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), columns, TABLE_COMMENT, createProperties(), Transforms.EMPTY_TRANSFORM); - NameIdentifier id3 = NameIdentifier.of(metalakeName, catalogName, schemaName, newTableName); - NameIdentifier id4 = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier id3 = NameIdentifier.of(schemaName, newTableName); + NameIdentifier id4 = NameIdentifier.of(schemaName, tableName); TableChange newRename = TableChange.rename(newTableName); TableChange oldRename = TableChange.rename(tableName); TableCatalog tableCatalog = catalog.asTableCatalog(); @@ -1468,7 +1458,7 @@ public void testDropHiveManagedTable() throws TException, InterruptedException, catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), columns, TABLE_COMMENT, createProperties(), @@ -1479,9 +1469,7 @@ public void testDropHiveManagedTable() throws TException, InterruptedException, checkTableReadWrite(hiveTab); Assertions.assertEquals(MANAGED_TABLE.name(), hiveTab.getTableType()); Path tableDirectory = new Path(hiveTab.getSd().getLocation()); - catalog - .asTableCatalog() - .dropTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName)); Boolean existed = hiveClientPool.run(client -> client.tableExists(schemaName, tableName)); Assertions.assertFalse(existed, "The hive table should not exist"); Assertions.assertFalse(hdfs.exists(tableDirectory), "The table directory should not exist"); @@ -1493,7 +1481,7 @@ public void testDropHiveExternalTable() throws TException, InterruptedException, catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), columns, TABLE_COMMENT, ImmutableMap.of(TABLE_TYPE, EXTERNAL_TABLE.name().toLowerCase(Locale.ROOT)), @@ -1503,9 +1491,7 @@ public void testDropHiveExternalTable() throws TException, InterruptedException, hiveClientPool.run(client -> client.getTable(schemaName, tableName)); checkTableReadWrite(hiveTab); Assertions.assertEquals(EXTERNAL_TABLE.name(), hiveTab.getTableType()); - catalog - .asTableCatalog() - .dropTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName)); Boolean existed = hiveClientPool.run(client -> client.tableExists(schemaName, tableName)); Assertions.assertFalse(existed, "The table should be not exist"); @@ -1520,7 +1506,7 @@ public void testPurgeHiveManagedTable() throws TException, InterruptedException, catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), columns, TABLE_COMMENT, createProperties(), @@ -1530,9 +1516,7 @@ public void testPurgeHiveManagedTable() throws TException, InterruptedException, hiveClientPool.run(client -> client.getTable(schemaName, tableName)); checkTableReadWrite(hiveTab); Assertions.assertEquals(MANAGED_TABLE.name(), hiveTab.getTableType()); - catalog - .asTableCatalog() - .purgeTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName)); Boolean existed = hiveClientPool.run(client -> client.tableExists(schemaName, tableName)); Assertions.assertFalse(existed, "The hive table should not exist"); Path tableDirectory = new Path(hiveTab.getSd().getLocation()); @@ -1547,7 +1531,7 @@ public void testPurgeHiveExternalTable() throws TException, InterruptedException catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), columns, TABLE_COMMENT, ImmutableMap.of(TABLE_TYPE, EXTERNAL_TABLE.name().toLowerCase(Locale.ROOT)), @@ -1558,7 +1542,7 @@ public void testPurgeHiveExternalTable() throws TException, InterruptedException checkTableReadWrite(hiveTab); Assertions.assertEquals(EXTERNAL_TABLE.name(), hiveTab.getTableType()); TableCatalog tableCatalog = catalog.asTableCatalog(); - NameIdentifier id = NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier id = NameIdentifier.of(schemaName, tableName); Assertions.assertThrows( UnsupportedOperationException.class, () -> { @@ -1579,7 +1563,7 @@ public void testRemoveNonExistTable() throws TException, InterruptedException { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), columns, TABLE_COMMENT, ImmutableMap.of(TABLE_TYPE, EXTERNAL_TABLE.name().toLowerCase(Locale.ROOT)), @@ -1594,15 +1578,11 @@ public void testRemoveNonExistTable() throws TException, InterruptedException { // Drop table from catalog, drop non-exist table should return false; Assertions.assertFalse( - catalog - .asTableCatalog() - .dropTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)), + catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName)), "The table should not be found in the catalog"); Assertions.assertFalse( - catalog - .asTableCatalog() - .tableExists(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)), + catalog.asTableCatalog().tableExists(NameIdentifier.of(schemaName, tableName)), "The table should not be found in the catalog"); } @@ -1612,7 +1592,7 @@ public void testPurgeNonExistTable() throws TException, InterruptedException { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), columns, TABLE_COMMENT, ImmutableMap.of(TABLE_TYPE, EXTERNAL_TABLE.name().toLowerCase(Locale.ROOT)), @@ -1627,15 +1607,11 @@ public void testPurgeNonExistTable() throws TException, InterruptedException { // Drop table from catalog, drop non-exist table should return false; Assertions.assertFalse( - catalog - .asTableCatalog() - .purgeTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)), + catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName)), "The table should not be found in the catalog"); Assertions.assertFalse( - catalog - .asTableCatalog() - .tableExists(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)), + catalog.asTableCatalog().tableExists(NameIdentifier.of(schemaName, tableName)), "The table should not be found in the catalog"); } diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java index 4880e3a7da6..d1e06d35a88 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive.integration.test; @@ -47,7 +61,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class HiveUserAuthenticationIT extends AbstractIT { private static final Logger LOG = LoggerFactory.getLogger(HiveUserAuthenticationIT.class); @@ -117,7 +131,6 @@ public static void stop() { System.clearProperty("java.security.krb5.conf"); System.clearProperty("sun.security.krb5.debug"); - AbstractIT.customConfigs.clear(); AbstractIT.client = null; } @@ -232,8 +245,7 @@ public void testUserAuthentication() { () -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", ImmutableMap.of())); // Create table - NameIdentifier tableNameIdentifier = - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, SCHEMA_NAME, TABLE_NAME); + NameIdentifier tableNameIdentifier = NameIdentifier.of(SCHEMA_NAME, TABLE_NAME); catalog .asTableCatalog() .createTable( @@ -247,8 +259,7 @@ public void testUserAuthentication() { // Now try to alter the table catalog.asTableCatalog().alterTable(tableNameIdentifier, TableChange.rename("new_table")); - NameIdentifier newTableIdentifier = - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, SCHEMA_NAME, "new_table"); + NameIdentifier newTableIdentifier = NameIdentifier.of(SCHEMA_NAME, "new_table"); // Old table name should not exist Assertions.assertFalse(catalog.asTableCatalog().tableExists(tableNameIdentifier)); diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java index fee4b49df3f..ee9305065ea 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.hive.integration.test; @@ -49,7 +63,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class ProxyCatalogHiveIT extends AbstractIT { public static final String METALAKE_NAME = @@ -119,7 +133,6 @@ public static void stop() { setEnv(HADOOP_USER_NAME, originHadoopUser); anotherClient.close(); - AbstractIT.customConfigs.clear(); AbstractIT.client = null; } @@ -129,10 +142,8 @@ public void testOperateSchema() throws Exception { String schemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX); String anotherSchemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX); - NameIdentifier ident = NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, schemaName); - String comment = "comment"; - createSchema(schemaName, ident, comment); + createSchema(schemaName, comment); Database db = hiveClientPool.run(client -> client.getDatabase(schemaName)); Assertions.assertEquals(EXPECT_USER, db.getOwnerName()); @@ -164,14 +175,11 @@ public void testOperateTable() throws Exception { String tableName = GravitinoITUtils.genRandomName(TABLE_PREFIX); String anotherTableName = GravitinoITUtils.genRandomName(TABLE_PREFIX); - NameIdentifier ident = NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, schemaName); - NameIdentifier nameIdentifier = - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, schemaName, tableName); - NameIdentifier anotherNameIdentifier = - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, schemaName, anotherTableName); + NameIdentifier nameIdentifier = NameIdentifier.of(schemaName, tableName); + NameIdentifier anotherNameIdentifier = NameIdentifier.of(schemaName, anotherTableName); String comment = "comment"; - createSchema(schemaName, ident, comment); + createSchema(schemaName, comment); Table createdTable = catalog @@ -201,7 +209,7 @@ public void testOperateTable() throws Exception { Assertions.assertTrue(e.getMessage().contains("AccessControlException Permission denied")); } - private static void createSchema(String schemaName, NameIdentifier ident, String comment) { + private static void createSchema(String schemaName, String comment) { Map properties = Maps.newHashMap(); properties.put("key1", "val1"); properties.put("key2", "val2"); @@ -212,7 +220,7 @@ private static void createSchema(String schemaName, NameIdentifier ident, String containerSuite.getHiveContainer().getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, schemaName.toLowerCase())); - catalog.asSchemas().createSchema(ident.name(), comment, properties); + catalog.asSchemas().createSchema(schemaName, comment, properties); } @Test @@ -222,12 +230,10 @@ public void testOperatePartition() throws Exception { String schemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX); String tableName = GravitinoITUtils.genRandomName(TABLE_PREFIX); - NameIdentifier ident = NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, schemaName); - NameIdentifier nameIdentifier = - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, schemaName, tableName); + NameIdentifier nameIdentifier = NameIdentifier.of(schemaName, tableName); String comment = "comment"; - createSchema(schemaName, ident, comment); + createSchema(schemaName, comment); // create a partitioned table Column[] columns = createColumns(); diff --git a/catalogs/catalog-hive/src/test/resources/hive-schema-3.1.0.derby.sql b/catalogs/catalog-hive/src/test/resources/hive-schema-3.1.0.derby.sql index 55097d6639f..02987cb147c 100644 --- a/catalogs/catalog-hive/src/test/resources/hive-schema-3.1.0.derby.sql +++ b/catalogs/catalog-hive/src/test/resources/hive-schema-3.1.0.derby.sql @@ -3,7 +3,7 @@ -- contributor license agreements. See the NOTICE file distributed with -- this work for additional information regarding copyright ownership. -- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with +-- (the "License"). You may not use this file except in compliance with -- the License. You may obtain a copy of the License at -- -- http://www.apache.org/licenses/LICENSE-2.0 diff --git a/catalogs/catalog-hive/src/test/resources/hive-site.xml b/catalogs/catalog-hive/src/test/resources/hive-site.xml index ce9e4f1fd9b..d3b285a6888 100644 --- a/catalogs/catalog-hive/src/test/resources/hive-site.xml +++ b/catalogs/catalog-hive/src/test/resources/hive-site.xml @@ -1,6 +1,20 @@ @@ -14,4 +28,3 @@ - diff --git a/catalogs/catalog-hive/src/test/resources/log4j2.properties b/catalogs/catalog-hive/src/test/resources/log4j2.properties index ab88dcde35d..112762fe35c 100644 --- a/catalogs/catalog-hive/src/test/resources/log4j2.properties +++ b/catalogs/catalog-hive/src/test/resources/log4j2.properties @@ -1,6 +1,20 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # Set to debug or trace if log4j initialization is failing diff --git a/catalogs/catalog-jdbc-common/build.gradle.kts b/catalogs/catalog-jdbc-common/build.gradle.kts index 1e884afe8eb..05200302d48 100644 --- a/catalogs/catalog-jdbc-common/build.gradle.kts +++ b/catalogs/catalog-jdbc-common/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ description = "catalog-jdbc-common" diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalog.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalog.java index 6c28fae23a1..a2041db12ff 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalog.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc; @@ -13,6 +27,7 @@ import com.datastrato.gravitino.connector.CatalogOperations; import com.datastrato.gravitino.connector.PropertiesMetadata; import com.datastrato.gravitino.connector.PropertyEntry; +import com.datastrato.gravitino.connector.capability.Capability; import java.util.Collections; import java.util.Map; @@ -52,6 +67,11 @@ protected CatalogOperations newOps(Map config) { return ops; } + @Override + public Capability newCapability() { + return new JdbcCatalogCapability(); + } + /** @return The {@link JdbcExceptionConverter} to be used by the catalog. */ protected JdbcExceptionConverter createExceptionConverter() { return new JdbcExceptionConverter() {}; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogCapability.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogCapability.java new file mode 100644 index 00000000000..75439269b74 --- /dev/null +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogCapability.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.jdbc; + +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.connector.capability.CapabilityResult; + +public class JdbcCatalogCapability implements Capability { + /** + * Regular expression explanation: Regex that matches any string that maybe a filename with an + * optional extension We adopt a blacklist approach that excludes filename or extension that + * contains '.', '/', or '\' ^[^.\/\\]+(\.[^.\/\\]+)?$ + * + *

^ - Start of the string + * + *

[^.\/\\]+ - matches any filename string that does not contain '.', '/', or '\' + * + *

(\.[^.\/\\]+)? - matches an optional extension + * + *

$ - End of the string + */ + // We use sqlite name pattern to be the default pattern for JDBC catalog for testing purposes + public static final String SQLITE_NAME_PATTERN = "^[^.\\/\\\\]+(\\.[^.\\/\\\\]+)?$"; + + @Override + public CapabilityResult specificationOnName(Scope scope, String name) { + // TODO: Validate the name against reserved words + if (!name.matches(SQLITE_NAME_PATTERN)) { + return CapabilityResult.unsupported( + String.format("The %s name '%s' is illegal.", scope, name)); + } + return CapabilityResult.SUPPORTED; + } +} diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java index 1fb3726bb2b..14ab7369192 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogPropertiesMetadata.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogPropertiesMetadata.java index c8cbb9fc84f..3ac8a745948 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogPropertiesMetadata.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogPropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcColumn.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcColumn.java index 5de0a4ab945..5bc491e68d5 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcColumn.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcColumn.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcSchema.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcSchema.java index e9e7e705d86..186180ba78b 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcSchema.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcSchema.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcSchemaPropertiesMetadata.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcSchemaPropertiesMetadata.java index 16899d9b72f..63e8340732a 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcSchemaPropertiesMetadata.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcSchemaPropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTable.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTable.java index 9180820633a..664be416b1b 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTable.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTable.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTablePropertiesMetadata.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTablePropertiesMetadata.java index 5b3b4eec8a6..216b0190c82 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTablePropertiesMetadata.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTablePropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java index 4811cde05a2..4ee47bacfe1 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/bean/JdbcIndexBean.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/bean/JdbcIndexBean.java index a6a9f12ca69..f29d3993722 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/bean/JdbcIndexBean.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/bean/JdbcIndexBean.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.bean; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java index de397d3eb18..a42a8a2ed29 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.config; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcColumnDefaultValueConverter.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcColumnDefaultValueConverter.java index 4c879535629..7aeca947a8c 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcColumnDefaultValueConverter.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcColumnDefaultValueConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.converter; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcExceptionConverter.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcExceptionConverter.java index 10817984022..29ab8f91aca 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcExceptionConverter.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcExceptionConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.converter; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcTypeConverter.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcTypeConverter.java index e9bc3443a49..8145e963913 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcTypeConverter.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/converter/JdbcTypeConverter.java @@ -1,14 +1,28 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.converter; -import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.connector.DataTypeConverter; import java.util.Objects; -/** @param Implement the corresponding JDBC data type to be converted */ -public abstract class JdbcTypeConverter { +public abstract class JdbcTypeConverter + implements DataTypeConverter { public static final String DATE = "date"; public static final String TIME = "time"; @@ -16,22 +30,6 @@ public abstract class JdbcTypeConverter { public static final String VARCHAR = "varchar"; public static final String TEXT = "text"; - /** - * Convert from JDBC type to Gravitino type - * - * @param type The common jdbc type bean. - * @return Gravitino type. - */ - public abstract Type toGravitinoType(JdbcTypeBean type); - - /** - * Convert from Gravitino type to JDBC type - * - * @param type Gravitino type. - * @return Implement the corresponding JDBC data type to be converted. - */ - public abstract TO fromGravitinoType(Type type); - public static class JdbcTypeBean { /** Data type name. */ private String typeName; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/DatabaseOperation.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/DatabaseOperation.java index db424b86809..2cc6b712857 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/DatabaseOperation.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/DatabaseOperation.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.operation; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcDatabaseOperations.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcDatabaseOperations.java index 7b13d4f3541..54aa76f15c9 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcDatabaseOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcDatabaseOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.operation; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java index 126e792905d..39b6860a1be 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.operation; @@ -456,7 +470,7 @@ protected JdbcColumn.Builder getBasicJdbcColumnInfo(ResultSet column) throws SQL return JdbcColumn.builder() .withName(column.getString("COLUMN_NAME")) - .withType(typeConverter.toGravitinoType(typeBean)) + .withType(typeConverter.toGravitino(typeBean)) .withComment(StringUtils.isEmpty(comment) ? null : comment) .withNullable(nullable) .withDefaultValue(defaultValue); diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/TableOperation.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/TableOperation.java index 8cd5bb68509..987d5720713 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/TableOperation.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/TableOperation.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.operation; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java index 95eaf682348..7bea43589dd 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.utils; diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/JdbcConnectorUtils.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/JdbcConnectorUtils.java index 209ccf5bd02..d17cb58bdff 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/JdbcConnectorUtils.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/JdbcConnectorUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.utils; diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/TestJdbc.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/TestJdbc.java index ca44e57e945..0cef368b2e1 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/TestJdbc.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/TestJdbc.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc; diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/config/TestJdbcConfig.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/config/TestJdbcConfig.java index 982bc75fb6e..b092d19b56f 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/config/TestJdbcConfig.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/config/TestJdbcConfig.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.config; diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteColumnDefaultValueConverter.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteColumnDefaultValueConverter.java index 3356737b121..dc95e27b4d0 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteColumnDefaultValueConverter.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteColumnDefaultValueConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.converter; diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteExceptionConverter.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteExceptionConverter.java index db59591d602..27d63c18bba 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteExceptionConverter.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteExceptionConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.converter; diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteTypeConverter.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteTypeConverter.java index 9e6ec75756f..9c42e4c4837 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteTypeConverter.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/SqliteTypeConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.converter; @@ -11,7 +25,7 @@ import java.util.Map; import org.apache.commons.lang3.StringUtils; -public class SqliteTypeConverter extends JdbcTypeConverter { +public class SqliteTypeConverter extends JdbcTypeConverter { protected static final Map GRAVITINO_TO_SQLITE_MAPPING = new HashMap<>(); @@ -23,7 +37,7 @@ public class SqliteTypeConverter extends JdbcTypeConverter { } @Override - public Type toGravitinoType(JdbcTypeBean type) { + public Type toGravitino(JdbcTypeBean type) { return GRAVITINO_TO_SQLITE_MAPPING.entrySet().stream() .filter(entry -> StringUtils.equalsIgnoreCase(type.getTypeName(), entry.getValue())) .map(Map.Entry::getKey) @@ -32,7 +46,7 @@ public Type toGravitinoType(JdbcTypeBean type) { } @Override - public String fromGravitinoType(Type type) { + public String fromGravitino(Type type) { return GRAVITINO_TO_SQLITE_MAPPING.get(type); } diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/TestJdbcColumnDefaultValueConverter.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/TestJdbcColumnDefaultValueConverter.java index fafa8c1edf8..c0de0e0ee87 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/TestJdbcColumnDefaultValueConverter.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/converter/TestJdbcColumnDefaultValueConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.converter; diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/SqliteDatabaseOperations.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/SqliteDatabaseOperations.java index 57f11669d67..912c7a5e984 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/SqliteDatabaseOperations.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/SqliteDatabaseOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.operation; diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/SqliteTableOperations.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/SqliteTableOperations.java index 66fa734ce08..20a79ae26b9 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/SqliteTableOperations.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/SqliteTableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.operation; @@ -42,7 +56,7 @@ protected String generateCreateTableSql( sqlBuilder .append(column.name()) .append(" ") - .append(typeConverter.fromGravitinoType(column.dataType())); + .append(typeConverter.fromGravitino(column.dataType())); if (!column.nullable()) { sqlBuilder.append(" NOT NULL"); } diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcDatabaseOperations.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcDatabaseOperations.java index 7dcc634ea94..0a109c28e9f 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcDatabaseOperations.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcDatabaseOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.operation; diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java index 27fbc9c4a1a..e472e07aeb6 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.operation; diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestDataSourceUtils.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestDataSourceUtils.java index a1946aff09f..82aaf77c516 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestDataSourceUtils.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestDataSourceUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.utils; diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestJdbcConfig.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestJdbcConfig.java index 3d7cbaa6413..3809901ec97 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestJdbcConfig.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestJdbcConfig.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.jdbc.utils; diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts index f884d512d4f..65d6e0d582c 100644 --- a/catalogs/catalog-jdbc-doris/build.gradle.kts +++ b/catalogs/catalog-jdbc-doris/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ description = "catalog-jdbc-doris" @@ -70,7 +84,7 @@ tasks { tasks.test { val skipUTs = project.hasProperty("skipTests") doFirst { - environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.3") + environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.4") } if (skipUTs) { diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalog.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalog.java index 30e25987bc2..68a9e5826ad 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalog.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris; @@ -17,6 +31,7 @@ import com.datastrato.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations; import com.datastrato.gravitino.catalog.jdbc.operation.JdbcTableOperations; import com.datastrato.gravitino.connector.CatalogOperations; +import com.datastrato.gravitino.connector.capability.Capability; import java.util.Map; /** Implementation of a Doris catalog in Gravitino. */ @@ -29,7 +44,7 @@ public String shortName() { @Override protected CatalogOperations newOps(Map config) { - JdbcTypeConverter jdbcTypeConverter = createJdbcTypeConverter(); + JdbcTypeConverter jdbcTypeConverter = createJdbcTypeConverter(); return new MySQLProtocolCompatibleCatalogOperations( createExceptionConverter(), jdbcTypeConverter, @@ -38,6 +53,11 @@ protected CatalogOperations newOps(Map config) { createJdbcColumnDefaultValueConverter()); } + @Override + public Capability newCapability() { + return new DorisCatalogCapability(); + } + @Override protected JdbcExceptionConverter createExceptionConverter() { return new DorisExceptionConverter(); diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalogCapability.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalogCapability.java new file mode 100644 index 00000000000..99d93a01c35 --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalogCapability.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.doris; + +import com.datastrato.gravitino.connector.capability.Capability; + +public class DorisCatalogCapability implements Capability { + // Doris best practice mention that the name should be in lowercase, separated by underscores + // https://doris.apache.org/docs/2.0/table-design/best-practice/ + // We can use the more general DEFAULT_NAME_PATTERN for Doris and update as needed in the future +} diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisColumnDefaultValueConverter.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisColumnDefaultValueConverter.java index b9b1b0f70b4..3b15379dc9c 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisColumnDefaultValueConverter.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisColumnDefaultValueConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.converter; diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisExceptionConverter.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisExceptionConverter.java index da272bd3287..de47c940025 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisExceptionConverter.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisExceptionConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.converter; diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisTypeConverter.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisTypeConverter.java index fa7fc1595f4..e2524982626 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisTypeConverter.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisTypeConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.converter; @@ -9,7 +23,7 @@ import com.datastrato.gravitino.rel.types.Types; /** Type converter for Doris. */ -public class DorisTypeConverter extends JdbcTypeConverter { +public class DorisTypeConverter extends JdbcTypeConverter { static final String BOOLEAN = "boolean"; static final String TINYINT = "tinyint"; static final String SMALLINT = "smallint"; @@ -23,7 +37,7 @@ public class DorisTypeConverter extends JdbcTypeConverter { static final String STRING = "string"; @Override - public Type toGravitinoType(JdbcTypeBean typeBean) { + public Type toGravitino(JdbcTypeBean typeBean) { switch (typeBean.getTypeName().toLowerCase()) { case BOOLEAN: return Types.BooleanType.get(); @@ -59,7 +73,7 @@ public Type toGravitinoType(JdbcTypeBean typeBean) { } @Override - public String fromGravitinoType(Type type) { + public String fromGravitino(Type type) { if (type instanceof Types.BooleanType) { return BOOLEAN; } else if (type instanceof Types.ByteType) { diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisDatabaseOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisDatabaseOperations.java index 17186209fd7..8bb787d6685 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisDatabaseOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisDatabaseOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.operation; diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java index 3a1100e576b..649f099552b 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.operation; @@ -523,7 +537,7 @@ private String updateColumnCommentFieldDefinition( } private String addColumnFieldDefinition(TableChange.AddColumn addColumn) { - String dataType = (String) typeConverter.fromGravitinoType(addColumn.getDataType()); + String dataType = typeConverter.fromGravitino(addColumn.getDataType()); if (addColumn.fieldName().length > 1) { throw new UnsupportedOperationException("Doris does not support nested column names."); } @@ -637,10 +651,7 @@ private String updateColumnTypeFieldDefinition( private StringBuilder appendColumnDefinition(JdbcColumn column, StringBuilder sqlBuilder) { // Add data type - sqlBuilder - .append(SPACE) - .append(typeConverter.fromGravitinoType(column.dataType())) - .append(SPACE); + sqlBuilder.append(SPACE).append(typeConverter.fromGravitino(column.dataType())).append(SPACE); // Add NOT NULL if the column is marked as such if (column.nullable()) { diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/utils/DorisUtils.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/utils/DorisUtils.java index 936d57a9c4b..91ebac3ce4c 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/utils/DorisUtils.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/utils/DorisUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.utils; diff --git a/catalogs/catalog-jdbc-doris/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider b/catalogs/catalog-jdbc-doris/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider index 94219d8a5d2..1c7eee86895 100644 --- a/catalogs/catalog-jdbc-doris/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider +++ b/catalogs/catalog-jdbc-doris/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider @@ -1,5 +1,19 @@ # -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # com.datastrato.gravitino.catalog.doris.DorisCatalog \ No newline at end of file diff --git a/catalogs/catalog-jdbc-doris/src/main/resources/jdbc-doris.conf b/catalogs/catalog-jdbc-doris/src/main/resources/jdbc-doris.conf index 75b0445be99..b58ec2968be 100644 --- a/catalogs/catalog-jdbc-doris/src/main/resources/jdbc-doris.conf +++ b/catalogs/catalog-jdbc-doris/src/main/resources/jdbc-doris.conf @@ -1,6 +1,20 @@ # -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # jdbc-url = jdbc:mysql://localhost:9030/ # jdbc-user = strato diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/converter/TestDorisExceptionConverter.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/converter/TestDorisExceptionConverter.java index 3ae51acdb82..13e1afadc3d 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/converter/TestDorisExceptionConverter.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/converter/TestDorisExceptionConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.converter; diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java index 03a36e63e3d..4fc88ecb7c7 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java @@ -1,12 +1,26 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.integration.test; import org.junit.jupiter.api.Tag; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class CatalogDorisDriverIT extends CatalogDorisIT { public CatalogDorisDriverIT() { super(); diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index 361c1bfc4d7..1a2744c6d71 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.integration.test; @@ -26,18 +40,22 @@ import com.datastrato.gravitino.rel.expressions.NamedReference; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.indexes.Index; import com.datastrato.gravitino.rel.indexes.Indexes; import com.datastrato.gravitino.rel.types.Types; +import com.datastrato.gravitino.utils.RandomNameUtils; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -48,7 +66,7 @@ import org.junit.jupiter.api.TestInstance.Lifecycle; import org.testcontainers.shaded.org.awaitility.Awaitility; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) public class CatalogDorisIT extends AbstractIT { @@ -227,7 +245,7 @@ void testDropDorisSchema() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), createColumns(), "Created by gravitino client", createTableProperties(), @@ -256,11 +274,69 @@ void testDropDorisSchema() { }); } + @Test + void testSchemaWithIllegalName() { + SupportsSchemas schemas = catalog.asSchemas(); + String databaseName = RandomNameUtils.genRandomName("it_db"); + Map properties = new HashMap<>(); + String comment = "comment"; + + // should throw an exception with string that might contain SQL injection + String sqlInjection = databaseName + "`; DROP TABLE important_table; -- "; + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.createSchema(sqlInjection, comment, properties); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.dropSchema(sqlInjection, false); + }); + + String sqlInjection1 = databaseName + "`; SLEEP(10); -- "; + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.createSchema(sqlInjection1, comment, properties); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.dropSchema(sqlInjection1, false); + }); + + String sqlInjection2 = + databaseName + "`; UPDATE Users SET password = 'newpassword' WHERE username = 'admin'; -- "; + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.createSchema(sqlInjection2, comment, properties); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.dropSchema(sqlInjection2, false); + }); + + // should throw an exception with input that has more than 64 characters + String invalidInput = StringUtils.repeat("a", 65); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.createSchema(invalidInput, comment, properties); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.dropSchema(invalidInput, false); + }); + } + @Test void testDorisTableBasicOperation() { // create a table - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Column[] columns = createColumns(); Distribution distribution = createDistribution(); @@ -294,18 +370,128 @@ void testDorisTableBasicOperation() { // rename table String newTableName = GravitinoITUtils.genRandomName("new_table_name"); tableCatalog.alterTable(tableIdentifier, TableChange.rename(newTableName)); - NameIdentifier newTableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, newTableName); + NameIdentifier newTableIdentifier = NameIdentifier.of(schemaName, newTableName); Table renamedTable = tableCatalog.loadTable(newTableIdentifier); ITUtils.assertionsTableInfo( newTableName, table_comment, Arrays.asList(columns), properties, indexes, renamedTable); } + @Test + void testDorisIllegalTableName() { + Map properties = createTableProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + String table_name = "t123"; + + String t1_name = table_name + "`; DROP TABLE important_table; -- "; + Column t1_col = Column.of(t1_name, Types.LongType.get(), "id", false, false, null); + Column[] columns = {t1_col}; + Index[] t1_indexes = {Indexes.unique("u1_key", new String[][] {{t1_name}})}; + NameIdentifier tableIdentifier = + NameIdentifier.of(metalakeName, catalogName, schemaName, t1_name); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t1_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier); + }); + + String t2_name = table_name + "`; SLEEP(10); -- "; + Column t2_col = Column.of(t2_name, Types.LongType.get(), "id", false, false, null); + Index[] t2_indexes = {Indexes.unique("u2_key", new String[][] {{t2_name}})}; + Column[] columns2 = new Column[] {t2_col}; + NameIdentifier tableIdentifier2 = + NameIdentifier.of(metalakeName, catalogName, schemaName, t2_name); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier2, + columns2, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t2_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier2); + }); + + String t3_name = + table_name + "`; UPDATE Users SET password = 'newpassword' WHERE username = 'admin'; -- "; + Column t3_col = Column.of(t3_name, Types.LongType.get(), "id", false, false, null); + Index[] t3_indexes = {Indexes.unique("u3_key", new String[][] {{t3_name}})}; + Column[] columns3 = new Column[] {t3_col}; + NameIdentifier tableIdentifier3 = + NameIdentifier.of(metalakeName, catalogName, schemaName, t3_name); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier3, + columns3, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t3_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier3); + }); + + String invalidInput = StringUtils.repeat("a", 65); + Column t4_col = Column.of(invalidInput, Types.LongType.get(), "id", false, false, null); + Index[] t4_indexes = {Indexes.unique("u4_key", new String[][] {{invalidInput}})}; + Column[] columns4 = new Column[] {t4_col}; + NameIdentifier tableIdentifier4 = + NameIdentifier.of(metalakeName, catalogName, schemaName, invalidInput); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier4, + columns4, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t4_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier4); + }); + } + @Test void testAlterDorisTable() { // create a table - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Column[] columns = createColumns(); Distribution distribution = createDistribution(); @@ -397,8 +583,7 @@ void testAlterDorisTable() { void testDorisIndex() { String tableName = GravitinoITUtils.genRandomName("test_add_index"); - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Column[] columns = createColumns(); Distribution distribution = createDistribution(); @@ -418,7 +603,7 @@ void testDorisIndex() { // add index test. tableCatalog.alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.addIndex( Index.IndexType.PRIMARY_KEY, "k1_index", new String[][] {{DORIS_COL_NAME1}})); @@ -430,14 +615,13 @@ void testDorisIndex() { assertEquals( 1, tableCatalog - .loadTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)) + .loadTable(NameIdentifier.of(schemaName, tableName)) .index() .length)); // delete index and add new column and index. tableCatalog.alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.deleteIndex("k1_index", true), TableChange.addIndex( Index.IndexType.PRIMARY_KEY, "k2_index", new String[][] {{DORIS_COL_NAME2}})); @@ -450,8 +634,7 @@ void testDorisIndex() { assertEquals( 1, tableCatalog - .loadTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)) + .loadTable(NameIdentifier.of(schemaName, tableName)) .index() .length)); } diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDoris.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDoris.java index 96ba1bb9bf7..840290ee5e5 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDoris.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDoris.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.operation; diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisDatabaseOperations.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisDatabaseOperations.java index 58a8a396f27..4483a02ca25 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisDatabaseOperations.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisDatabaseOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.operation; @@ -13,7 +27,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class TestDorisDatabaseOperations extends TestDoris { @Test diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java index 5c1c94c137b..e828956d320 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.operation; @@ -32,7 +46,7 @@ import org.junit.jupiter.api.Test; import org.testcontainers.shaded.org.awaitility.Awaitility; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class TestDorisTableOperations extends TestDoris { private static final Type VARCHAR_255 = Types.VarCharType.of(255); private static final Type VARCHAR_1024 = Types.VarCharType.of(1024); diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/utils/TestDorisUtils.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/utils/TestDorisUtils.java index 35923bbfd36..69d8a717e12 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/utils/TestDorisUtils.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/utils/TestDorisUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.doris.utils; diff --git a/catalogs/catalog-jdbc-doris/src/test/resources/log4j2.properties b/catalogs/catalog-jdbc-doris/src/test/resources/log4j2.properties index c8307ac956f..0ecbcf1f127 100644 --- a/catalogs/catalog-jdbc-doris/src/test/resources/log4j2.properties +++ b/catalogs/catalog-jdbc-doris/src/test/resources/log4j2.properties @@ -1,6 +1,20 @@ # -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # Set to debug or trace if log4j initialization is failing diff --git a/catalogs/catalog-jdbc-mysql/build.gradle.kts b/catalogs/catalog-jdbc-mysql/build.gradle.kts index 06f061ced86..9be41612aa2 100644 --- a/catalogs/catalog-jdbc-mysql/build.gradle.kts +++ b/catalogs/catalog-jdbc-mysql/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ description = "catalog-jdbc-mysql" diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalog.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalog.java index 69ee53a0dd0..901cb6b2828 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalog.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql; @@ -18,6 +32,7 @@ import com.datastrato.gravitino.catalog.mysql.operation.MysqlTableOperations; import com.datastrato.gravitino.connector.CatalogOperations; import com.datastrato.gravitino.connector.PropertiesMetadata; +import com.datastrato.gravitino.connector.capability.Capability; import java.util.Map; /** Implementation of a Mysql catalog in Gravitino. */ @@ -33,7 +48,7 @@ public String shortName() { @Override protected CatalogOperations newOps(Map config) { - JdbcTypeConverter jdbcTypeConverter = createJdbcTypeConverter(); + JdbcTypeConverter jdbcTypeConverter = createJdbcTypeConverter(); return new MySQLProtocolCompatibleCatalogOperations( createExceptionConverter(), jdbcTypeConverter, @@ -42,13 +57,18 @@ protected CatalogOperations newOps(Map config) { createJdbcColumnDefaultValueConverter()); } + @Override + public Capability newCapability() { + return new MysqlCatalogCapability(); + } + @Override protected JdbcExceptionConverter createExceptionConverter() { return new MysqlExceptionConverter(); } @Override - protected JdbcTypeConverter createJdbcTypeConverter() { + protected JdbcTypeConverter createJdbcTypeConverter() { return new MysqlTypeConverter(); } diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalogCapability.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalogCapability.java new file mode 100644 index 00000000000..905d0caf6fe --- /dev/null +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlCatalogCapability.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.mysql; + +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.connector.capability.CapabilityResult; + +public class MysqlCatalogCapability implements Capability { + /** + * Regular expression explanation: ^[\w\p{L}-$/=]{1,64}$ + * + *

^ - Start of the string + * + *

[\w\p{L}-$/=]{1,64} - Consist of 1 to 64 characters of letters (both cases), digits, + * underscores, any kind of letter from any language, hyphens, dollar signs, slashes or equal + * signs + * + *

\w - matches [a-zA-Z0-9_] + * + *

\p{L} - matches any kind of letter from any language + * + *

$ - End of the string + */ + public static final String MYSQL_NAME_PATTERN = "^[\\w\\p{L}-$/=]{1,64}$"; + + @Override + public CapabilityResult specificationOnName(Scope scope, String name) { + // TODO: Validate the name against reserved words + if (!name.matches(MYSQL_NAME_PATTERN)) { + return CapabilityResult.unsupported( + String.format("The %s name '%s' is illegal.", scope, name)); + } + return CapabilityResult.SUPPORTED; + } +} diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlTablePropertiesMetadata.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlTablePropertiesMetadata.java index 1162f048b2e..056279d8304 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlTablePropertiesMetadata.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlTablePropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql; diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlColumnDefaultValueConverter.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlColumnDefaultValueConverter.java index b34687d095c..c2b0092793f 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlColumnDefaultValueConverter.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlColumnDefaultValueConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.converter; diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlExceptionConverter.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlExceptionConverter.java index 3d89f3cfbc4..89fd3f171ca 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlExceptionConverter.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlExceptionConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.converter; diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlTypeConverter.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlTypeConverter.java index b82f4629a08..47ae3b2fec9 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlTypeConverter.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlTypeConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.converter; @@ -9,12 +23,16 @@ import com.datastrato.gravitino.rel.types.Types; /** Type converter for MySQL. */ -public class MysqlTypeConverter extends JdbcTypeConverter { +public class MysqlTypeConverter extends JdbcTypeConverter { static final String TINYINT = "tinyint"; + static final String TINYINT_UNSIGNED = "tinyint unsigned"; static final String SMALLINT = "smallint"; + static final String SMALLINT_UNSIGNED = "smallint unsigned"; static final String INT = "int"; + static final String INT_UNSIGNED = "int unsigned"; static final String BIGINT = "bigint"; + static final String BIGINT_UNSIGNED = "bigint unsigned"; static final String FLOAT = "float"; static final String DOUBLE = "double"; static final String DECIMAL = "decimal"; @@ -23,16 +41,24 @@ public class MysqlTypeConverter extends JdbcTypeConverter { static final String DATETIME = "datetime"; @Override - public Type toGravitinoType(JdbcTypeBean typeBean) { + public Type toGravitino(JdbcTypeBean typeBean) { switch (typeBean.getTypeName().toLowerCase()) { case TINYINT: return Types.ByteType.get(); + case TINYINT_UNSIGNED: + return Types.ByteType.unsigned(); case SMALLINT: return Types.ShortType.get(); + case SMALLINT_UNSIGNED: + return Types.ShortType.unsigned(); case INT: return Types.IntegerType.get(); + case INT_UNSIGNED: + return Types.IntegerType.unsigned(); case BIGINT: return Types.LongType.get(); + case BIGINT_UNSIGNED: + return Types.LongType.unsigned(); case FLOAT: return Types.FloatType.get(); case DOUBLE: @@ -66,15 +92,31 @@ public Type toGravitinoType(JdbcTypeBean typeBean) { } @Override - public String fromGravitinoType(Type type) { + public String fromGravitino(Type type) { if (type instanceof Types.ByteType) { - return TINYINT; + if (((Types.ByteType) type).signed()) { + return TINYINT; + } else { + return TINYINT_UNSIGNED; + } } else if (type instanceof Types.ShortType) { - return SMALLINT; + if (((Types.ShortType) type).signed()) { + return SMALLINT; + } else { + return SMALLINT_UNSIGNED; + } } else if (type instanceof Types.IntegerType) { - return INT; + if (((Types.IntegerType) type).signed()) { + return INT; + } else { + return INT_UNSIGNED; + } } else if (type instanceof Types.LongType) { - return BIGINT; + if (((Types.LongType) type).signed()) { + return BIGINT; + } else { + return BIGINT_UNSIGNED; + } } else if (type instanceof Types.FloatType) { return type.simpleString(); } else if (type instanceof Types.DoubleType) { diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlDatabaseOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlDatabaseOperations.java index 2c574edbf99..c2385f1b219 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlDatabaseOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlDatabaseOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.operation; diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java index 0446713c77b..64d4b98e144 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.operation; @@ -508,7 +522,7 @@ private String updateColumnCommentFieldDefinition( } private String addColumnFieldDefinition(TableChange.AddColumn addColumn) { - String dataType = (String) typeConverter.fromGravitinoType(addColumn.getDataType()); + String dataType = typeConverter.fromGravitino(addColumn.getDataType()); if (addColumn.fieldName().length > 1) { throw new UnsupportedOperationException(MYSQL_NOT_SUPPORT_NESTED_COLUMN_MSG); } @@ -683,10 +697,7 @@ private String updateColumnTypeFieldDefinition( private StringBuilder appendColumnDefinition(JdbcColumn column, StringBuilder sqlBuilder) { // Add data type - sqlBuilder - .append(SPACE) - .append(typeConverter.fromGravitinoType(column.dataType())) - .append(SPACE); + sqlBuilder.append(SPACE).append(typeConverter.fromGravitino(column.dataType())).append(SPACE); // Add NOT NULL if the column is marked as such if (column.nullable()) { diff --git a/catalogs/catalog-jdbc-mysql/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider b/catalogs/catalog-jdbc-mysql/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider index 59d06bb1335..359fd3b85a6 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider +++ b/catalogs/catalog-jdbc-mysql/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider @@ -1,5 +1,19 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # com.datastrato.gravitino.catalog.mysql.MysqlCatalog \ No newline at end of file diff --git a/catalogs/catalog-jdbc-mysql/src/main/resources/jdbc-mysql.conf b/catalogs/catalog-jdbc-mysql/src/main/resources/jdbc-mysql.conf index c086803ed4b..cca42d6db69 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/resources/jdbc-mysql.conf +++ b/catalogs/catalog-jdbc-mysql/src/main/resources/jdbc-mysql.conf @@ -1,6 +1,20 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # jdbc-url = jdbc:mysql://localhost:3306/ # jdbc-user = strato diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/converter/TestMysqlTypeConverter.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/converter/TestMysqlTypeConverter.java index 62aa8c0ed44..831c7131bf8 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/converter/TestMysqlTypeConverter.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/converter/TestMysqlTypeConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.converter; @@ -70,17 +84,17 @@ public void testFromGravitinoType() { checkGravitinoTypeToJdbcType(USER_DEFINED_TYPE, Types.ExternalType.of(USER_DEFINED_TYPE)); Assertions.assertThrows( IllegalArgumentException.class, - () -> MYSQL_TYPE_CONVERTER.fromGravitinoType(Types.UnparsedType.of(USER_DEFINED_TYPE))); + () -> MYSQL_TYPE_CONVERTER.fromGravitino(Types.UnparsedType.of(USER_DEFINED_TYPE))); } protected void checkGravitinoTypeToJdbcType(String jdbcTypeName, Type gravitinoType) { - Assertions.assertEquals(jdbcTypeName, MYSQL_TYPE_CONVERTER.fromGravitinoType(gravitinoType)); + Assertions.assertEquals(jdbcTypeName, MYSQL_TYPE_CONVERTER.fromGravitino(gravitinoType)); } protected void checkJdbcTypeToGravitinoType( Type gravitinoType, String jdbcTypeName, String columnSize, String scale) { JdbcTypeConverter.JdbcTypeBean typeBean = createTypeBean(jdbcTypeName, columnSize, scale); - Assertions.assertEquals(gravitinoType, MYSQL_TYPE_CONVERTER.toGravitinoType(typeBean)); + Assertions.assertEquals(gravitinoType, MYSQL_TYPE_CONVERTER.toGravitino(typeBean)); } protected static JdbcTypeConverter.JdbcTypeBean createTypeBean( diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java index 9929b2669d4..ba8e2b5b45f 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.integration.test; @@ -35,7 +49,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class AuditCatalogMysqlIT extends AbstractIT { private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); public static final String metalakeName = GravitinoITUtils.genRandomName("audit_mysql_metalake"); @@ -113,7 +127,7 @@ public void testAuditTable() throws Exception { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), new Column[] {col1}, "comment", properties); @@ -123,14 +137,12 @@ public void testAuditTable() throws Exception { catalog .asTableCatalog() .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.addColumn(new String[] {"col_4"}, Types.StringType.get())); Assertions.assertEquals(expectUser, table.auditInfo().creator()); Assertions.assertEquals(expectUser, table.auditInfo().lastModifier()); - catalog - .asTableCatalog() - .dropTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName)); catalog.asSchemas().dropSchema(schemaName, true); metalake.dropCatalog(catalogName); } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java index 910f31a56e4..ca32d6ac55c 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java @@ -1,13 +1,26 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.integration.test; import org.junit.jupiter.api.Tag; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class CatalogMysqlDriverIT extends CatalogMysqlIT { public CatalogMysqlDriverIT() { super(); diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java index ba722cc462a..00c560d044b 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.integration.test; @@ -43,6 +57,7 @@ import com.datastrato.gravitino.rel.indexes.Indexes; import com.datastrato.gravitino.rel.types.Decimal; import com.datastrato.gravitino.rel.types.Types; +import com.datastrato.gravitino.utils.RandomNameUtils; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -50,6 +65,7 @@ import java.sql.SQLException; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -65,7 +81,7 @@ import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.api.condition.EnabledIf; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) public class CatalogMysqlIT extends AbstractIT { private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); @@ -138,7 +154,7 @@ public void resetSchema() { private void clearTableAndSchema() { NameIdentifier[] nameIdentifiers = - catalog.asTableCatalog().listTables(Namespace.of(metalakeName, catalogName, schemaName)); + catalog.asTableCatalog().listTables(Namespace.of(schemaName)); for (NameIdentifier nameIdentifier : nameIdentifiers) { catalog.asTableCatalog().dropTable(nameIdentifier); } @@ -181,11 +197,10 @@ private void createCatalog() throws SQLException { } private void createSchema() { - NameIdentifier ident = NameIdentifier.of(metalakeName, catalogName, schemaName); Map prop = Maps.newHashMap(); - Schema createdSchema = catalog.asSchemas().createSchema(ident.name(), schema_comment, prop); - Schema loadSchema = catalog.asSchemas().loadSchema(ident.name()); + Schema createdSchema = catalog.asSchemas().createSchema(schemaName, schema_comment, prop); + Schema loadSchema = catalog.asSchemas().loadSchema(schemaName); Assertions.assertEquals(createdSchema.name(), loadSchema.name()); prop.forEach((key, value) -> Assertions.assertEquals(loadSchema.properties().get(key), value)); } @@ -256,7 +271,7 @@ void testOperationMysqlSchema() { // create schema check. String testSchemaName = GravitinoITUtils.genRandomName("test_schema_1"); NameIdentifier schemaIdent = NameIdentifier.of(metalakeName, catalogName, testSchemaName); - schemas.createSchema(schemaIdent.name(), schema_comment, Collections.emptyMap()); + schemas.createSchema(testSchemaName, schema_comment, Collections.emptyMap()); nameIdentifiers = schemas.listSchemas(); schemaNames = Sets.newHashSet(nameIdentifiers); Assertions.assertTrue(schemaNames.contains(testSchemaName)); @@ -270,13 +285,12 @@ void testOperationMysqlSchema() { Assertions.assertThrows( SchemaAlreadyExistsException.class, () -> { - schemas.createSchema(schemaIdent.name(), schema_comment, emptyMap); + schemas.createSchema(testSchemaName, schema_comment, emptyMap); }); // drop schema check. - schemas.dropSchema(schemaIdent.name(), false); - Assertions.assertThrows( - NoSuchSchemaException.class, () -> schemas.loadSchema(schemaIdent.name())); + schemas.dropSchema(testSchemaName, false); + Assertions.assertThrows(NoSuchSchemaException.class, () -> schemas.loadSchema(testSchemaName)); Assertions.assertThrows( NoSuchSchemaException.class, () -> mysqlService.loadSchema(schemaIdent)); @@ -287,8 +301,7 @@ void testOperationMysqlSchema() { TableCatalog tableCatalog = catalog.asTableCatalog(); // create failed check. - NameIdentifier table = - NameIdentifier.of(metalakeName, catalogName, testSchemaName, "test_table"); + NameIdentifier table = NameIdentifier.of(testSchemaName, "test_table"); Assertions.assertThrows( NoSuchSchemaException.class, () -> @@ -315,8 +328,7 @@ void testCreateAndLoadMysqlTable() { // Create table from Gravitino API Column[] columns = createColumns(); - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Distribution distribution = Distributions.NONE; final SortOrder[] sortOrders = new SortOrder[0]; @@ -374,7 +386,7 @@ void testColumnNameWithKeyWords() { }; String name = GravitinoITUtils.genRandomName("table") + "_keyword"; - NameIdentifier tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, name); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, name); Distribution distribution = Distributions.NONE; final SortOrder[] sortOrders = new SortOrder[0]; @@ -441,11 +453,7 @@ void testColumnDefaultValue() { catalog .asTableCatalog() .createTable( - NameIdentifier.of( - metalakeName, - catalogName, - schemaName, - GravitinoITUtils.genRandomName("mysql_it_table")), + NameIdentifier.of(schemaName, GravitinoITUtils.genRandomName("mysql_it_table")), newColumns, null, ImmutableMap.of()); @@ -500,9 +508,7 @@ void testColumnDefaultValueConverter() { mysqlService.executeQuery(sql); Table loadedTable = - catalog - .asTableCatalog() - .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName, tableName)); for (Column column : loadedTable.columns()) { switch (column.name()) { @@ -597,9 +603,7 @@ void testColumnTypeConverter() { mysqlService.executeQuery(sql); Table loadedTable = - catalog - .asTableCatalog() - .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName, tableName)); for (Column column : loadedTable.columns()) { switch (column.name()) { @@ -660,40 +664,32 @@ void testAlterAndDropMysqlTable() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), - columns, - table_comment, - createProperties()); + NameIdentifier.of(schemaName, tableName), columns, table_comment, createProperties()); Assertions.assertThrows( IllegalArgumentException.class, () -> { catalog .asTableCatalog() .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.rename(alertTableName), TableChange.updateComment(table_comment + "_new")); }); catalog .asTableCatalog() - .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), - TableChange.rename(alertTableName)); + .alterTable(NameIdentifier.of(schemaName, tableName), TableChange.rename(alertTableName)); catalog .asTableCatalog() .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, alertTableName), + NameIdentifier.of(schemaName, alertTableName), TableChange.updateComment(table_comment + "_new"), TableChange.addColumn(new String[] {"col_4"}, Types.StringType.get()), TableChange.renameColumn(new String[] {MYSQL_COL_NAME2}, "col_2_new"), TableChange.updateColumnType(new String[] {MYSQL_COL_NAME1}, Types.IntegerType.get())); - Table table = - catalog - .asTableCatalog() - .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, alertTableName)); + Table table = catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName, alertTableName)); Assertions.assertEquals(alertTableName, table.name()); Assertions.assertEquals(MYSQL_COL_NAME1, table.columns()[0].name()); @@ -722,11 +718,7 @@ void testAlterAndDropMysqlTable() { Column[] newColumns = new Column[] {col1, col2, col3}; NameIdentifier tableIdentifier = - NameIdentifier.of( - metalakeName, - catalogName, - schemaName, - GravitinoITUtils.genRandomName("CatalogJdbcIT_table")); + NameIdentifier.of(schemaName, GravitinoITUtils.genRandomName("CatalogJdbcIT_table")); catalog .asTableCatalog() .createTable( @@ -787,17 +779,14 @@ void testUpdateColumnDefaultValue() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), - columns, - null, - ImmutableMap.of()); + NameIdentifier.of(schemaName, tableName), columns, null, ImmutableMap.of()); Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().creator()); Assertions.assertNull(table.auditInfo().lastModifier()); catalog .asTableCatalog() .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.updateColumnDefaultValue( new String[] {columns[0].name()}, Literals.of("1.2345", Types.FloatType.get())), TableChange.updateColumnDefaultValue( @@ -809,10 +798,7 @@ void testUpdateColumnDefaultValue() { TableChange.updateColumnDefaultValue( new String[] {columns[4].name()}, Literals.of("2.34", Types.DecimalType.of(3, 2)))); - table = - catalog - .asTableCatalog() - .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + table = catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName, tableName)); Assertions.assertEquals( Literals.of("1.2345", Types.FloatType.get()), table.columns()[0].defaultValue()); @@ -838,7 +824,7 @@ void testDropMySQLDatabase() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), createColumns(), "Created by gravitino client", ImmutableMap.builder().build()); @@ -884,8 +870,7 @@ void testCreateTableIndex() { Indexes.unique("u6_key", new String[][] {{"col_3"}, {"col_4"}, {"col_1"}, {"col_2"}}), }; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Map properties = createProperties(); TableCatalog tableCatalog = catalog.asTableCatalog(); @@ -905,7 +890,7 @@ void testCreateTableIndex() { ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), properties, indexes, table); - NameIdentifier id = NameIdentifier.of(metalakeName, catalogName, schemaName, "test_failed"); + NameIdentifier id = NameIdentifier.of(schemaName, "test_failed"); Index[] indexes2 = new Index[] {Indexes.createMysqlPrimaryKey(new String[][] {{"col_1", "col_2"}})}; SortOrder[] sortOrder = new SortOrder[0]; @@ -950,7 +935,7 @@ void testCreateTableIndex() { table = tableCatalog.createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, "test_null_key"), + NameIdentifier.of(schemaName, "test_null_key"), newColumns, table_comment, properties, @@ -984,8 +969,7 @@ public void testAutoIncrement() { Indexes.unique("u1_key", new String[][] {{"col_2"}, {"col_3"}}) }; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Map properties = createProperties(); TableCatalog tableCatalog = catalog.asTableCatalog(); @@ -1131,8 +1115,7 @@ public void testBackQuoteTable() { Column col4 = Column.of("status", Types.VarCharType.of(255), "code", false, false, null); Column[] newColumns = new Column[] {col1, col2, col3, col4}; TableCatalog tableCatalog = catalog.asTableCatalog(); - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, "table"); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, "table"); Assertions.assertDoesNotThrow( () -> tableCatalog.createTable( @@ -1175,8 +1158,7 @@ void testMySQLSpecialTableName() { Index[] t1_indexes = {Indexes.unique("u1_key", new String[][] {{t1_name}})}; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, t1_name); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, t1_name); tableCatalog.createTable( tableIdentifier, columns, @@ -1191,7 +1173,7 @@ void testMySQLSpecialTableName() { Column t2_col = Column.of(t2_name, Types.LongType.get(), "id", false, false, null); Index[] t2_indexes = {Indexes.unique("u2_key", new String[][] {{t2_name}})}; columns = new Column[] {t2_col}; - tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, t2_name); + tableIdentifier = NameIdentifier.of(schemaName, t2_name); tableCatalog.createTable( tableIdentifier, columns, @@ -1206,7 +1188,7 @@ void testMySQLSpecialTableName() { Column t3_col = Column.of(t3_name, Types.LongType.get(), "id", false, false, null); Index[] t3_indexes = {Indexes.unique("u3_key", new String[][] {{t3_name}})}; columns = new Column[] {t3_col}; - tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, t3_name); + tableIdentifier = NameIdentifier.of(schemaName, t3_name); tableCatalog.createTable( tableIdentifier, columns, @@ -1221,7 +1203,7 @@ void testMySQLSpecialTableName() { Column t4_col = Column.of(t4_name, Types.LongType.get(), "id", false, false, null); Index[] t4_indexes = {Indexes.unique("u4_key", new String[][] {{t4_name}})}; columns = new Column[] {t4_col}; - tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, t4_name); + tableIdentifier = NameIdentifier.of(schemaName, t4_name); tableCatalog.createTable( tableIdentifier, columns, @@ -1232,31 +1214,139 @@ void testMySQLSpecialTableName() { new SortOrder[0], t4_indexes); - Table t1 = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t1_name)); + Table t1 = tableCatalog.loadTable(NameIdentifier.of(schemaName, t1_name)); Arrays.stream(t1.columns()).anyMatch(c -> Objects.equals(c.name(), "t112")); ITUtils.assertionsTableInfo( t1_name, table_comment, Arrays.asList(t1_col), properties, t1_indexes, t1); - Table t2 = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t2_name)); + Table t2 = tableCatalog.loadTable(NameIdentifier.of(schemaName, t2_name)); Arrays.stream(t2.columns()).anyMatch(c -> Objects.equals(c.name(), "t212")); ITUtils.assertionsTableInfo( t2_name, table_comment, Arrays.asList(t2_col), properties, t2_indexes, t2); - Table t3 = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t3_name)); + Table t3 = tableCatalog.loadTable(NameIdentifier.of(schemaName, t3_name)); Arrays.stream(t3.columns()).anyMatch(c -> Objects.equals(c.name(), "t_12")); ITUtils.assertionsTableInfo( t3_name, table_comment, Arrays.asList(t3_col), properties, t3_indexes, t3); - Table t4 = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t4_name)); + Table t4 = tableCatalog.loadTable(NameIdentifier.of(schemaName, t4_name)); Arrays.stream(t4.columns()).anyMatch(c -> Objects.equals(c.name(), "_1__")); ITUtils.assertionsTableInfo( t4_name, table_comment, Arrays.asList(t4_col), properties, t4_indexes, t4); } + @Test + void testMySqlIllegalTableName() { + Map properties = createProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + String table_name = "t123"; + + String t1_name = table_name + "`; DROP TABLE important_table; -- "; + Column t1_col = Column.of(t1_name, Types.LongType.get(), "id", false, false, null); + Column[] columns = {t1_col}; + Index[] t1_indexes = {Indexes.unique("u1_key", new String[][] {{t1_name}})}; + NameIdentifier tableIdentifier = + NameIdentifier.of(metalakeName, catalogName, schemaName, t1_name); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t1_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier); + }); + + String t2_name = table_name + "`; SLEEP(10); -- "; + Column t2_col = Column.of(t2_name, Types.LongType.get(), "id", false, false, null); + Index[] t2_indexes = {Indexes.unique("u2_key", new String[][] {{t2_name}})}; + Column[] columns2 = new Column[] {t2_col}; + NameIdentifier tableIdentifier2 = + NameIdentifier.of(metalakeName, catalogName, schemaName, t2_name); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier2, + columns2, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t2_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier2); + }); + + String t3_name = + table_name + "`; UPDATE Users SET password = 'newpassword' WHERE username = 'admin'; -- "; + Column t3_col = Column.of(t3_name, Types.LongType.get(), "id", false, false, null); + Index[] t3_indexes = {Indexes.unique("u3_key", new String[][] {{t3_name}})}; + Column[] columns3 = new Column[] {t3_col}; + NameIdentifier tableIdentifier3 = + NameIdentifier.of(metalakeName, catalogName, schemaName, t3_name); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier3, + columns3, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t3_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier3); + }); + + String invalidInput = StringUtils.repeat("a", 65); + Column t4_col = Column.of(invalidInput, Types.LongType.get(), "id", false, false, null); + Index[] t4_indexes = {Indexes.unique("u4_key", new String[][] {{invalidInput}})}; + Column[] columns4 = new Column[] {t4_col}; + NameIdentifier tableIdentifier4 = + NameIdentifier.of(metalakeName, catalogName, schemaName, invalidInput); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier4, + columns4, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t4_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier4); + }); + } + @Test void testMySQLTableNameCaseSensitive() { Column col1 = Column.of("col_1", Types.LongType.get(), "id", false, false, null); @@ -1272,8 +1362,7 @@ void testMySQLTableNameCaseSensitive() { Indexes.unique("u1_key", new String[][] {{"col_2"}, {"col_3"}}) }; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, "tableName"); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, "tableName"); Map properties = createProperties(); TableCatalog tableCatalog = catalog.asTableCatalog(); Table createdTable = @@ -1293,8 +1382,7 @@ void testMySQLTableNameCaseSensitive() { "tableName", table_comment, Arrays.asList(newColumns), properties, indexes, table); // Test create table with same name but different case - NameIdentifier tableIdentifier2 = - NameIdentifier.of(metalakeName, catalogName, schemaName, "TABLENAME"); + NameIdentifier tableIdentifier2 = NameIdentifier.of(schemaName, "TABLENAME"); Table tableAgain = Assertions.assertDoesNotThrow( @@ -1336,22 +1424,79 @@ void testNameSpec() { String testTableName = "//"; sql = String.format("CREATE TABLE `%s`.`%s` (id int)", testSchemaName, testTableName); mysqlService.executeQuery(sql); - NameIdentifier tableIdent = - NameIdentifier.of(metalakeName, catalogName, testSchemaName, testTableName); + NameIdentifier tableIdent = NameIdentifier.of(testSchemaName, testTableName); Table table = catalog.asTableCatalog().loadTable(tableIdent); Assertions.assertEquals(testTableName, table.name()); NameIdentifier[] tableIdents = - catalog - .asTableCatalog() - .listTables(Namespace.of(metalakeName, catalogName, testSchemaName)); + catalog.asTableCatalog().listTables(Namespace.of(testSchemaName)); Assertions.assertTrue(Arrays.stream(tableIdents).anyMatch(t -> t.name().equals(testTableName))); Assertions.assertTrue(catalog.asTableCatalog().dropTable(tableIdent)); Assertions.assertFalse(catalog.asTableCatalog().tableExists(tableIdent)); - Assertions.assertFalse(catalog.asTableCatalog().purgeTable(tableIdent)); + Assertions.assertThrows( + UnsupportedOperationException.class, + () -> { + catalog.asTableCatalog().purgeTable(tableIdent); + }); catalog.asSchemas().dropSchema(testSchemaName, true); + + // sql injection + String schemaName = RandomNameUtils.genRandomName("ct_db"); + Map properties = new HashMap<>(); + String comment = null; + + // should throw an exception with string that might contain SQL injection + String sqlInjection = schemaName + "`; DROP TABLE important_table; -- "; + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asSchemas().createSchema(sqlInjection, comment, properties); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asSchemas().dropSchema(sqlInjection, false); + }); + + String sqlInjection1 = schemaName + "`; SLEEP(10); -- "; + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asSchemas().createSchema(sqlInjection1, comment, properties); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asSchemas().dropSchema(sqlInjection1, false); + }); + + String sqlInjection2 = + schemaName + "`; UPDATE Users SET password = 'newpassword' WHERE username = 'admin'; -- "; + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asSchemas().createSchema(sqlInjection2, comment, properties); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asSchemas().dropSchema(sqlInjection2, false); + }); + + // should throw an exception with input that has more than 64 characters + String invalidInput = StringUtils.repeat("a", 65); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asSchemas().createSchema(invalidInput, comment, properties); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asSchemas().dropSchema(invalidInput, false); + }); } @Test @@ -1381,7 +1526,7 @@ void testMySQLSchemaNameCaseSensitive() { for (String schema : schemas) { tableCatalog.createTable( - NameIdentifier.of(metalakeName, catalogName, schema, tableName), + NameIdentifier.of(schema, tableName), newColumns, table_comment, properties, @@ -1390,8 +1535,7 @@ void testMySQLSchemaNameCaseSensitive() { new SortOrder[0], indexes); tableCatalog.createTable( - NameIdentifier.of( - metalakeName, catalogName, schema, GravitinoITUtils.genRandomName("test2")), + NameIdentifier.of(schema, GravitinoITUtils.genRandomName("test2")), newColumns, table_comment, properties, @@ -1402,8 +1546,7 @@ void testMySQLSchemaNameCaseSensitive() { } for (String schema : schemas) { - NameIdentifier[] nameIdentifiers = - tableCatalog.listTables(Namespace.of(metalakeName, catalogName, schema)); + NameIdentifier[] nameIdentifiers = tableCatalog.listTables(Namespace.of(schema)); Assertions.assertEquals(2, nameIdentifiers.length); Assertions.assertTrue( Arrays.stream(nameIdentifiers) @@ -1424,9 +1567,7 @@ void testUnparsedTypeConverter() { mysqlService.executeQuery( String.format("CREATE TABLE %s.%s (bit_col bit);", schemaName, tableName)); Table loadedTable = - catalog - .asTableCatalog() - .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName, tableName)); Assertions.assertEquals(Types.ExternalType.of("BIT"), loadedTable.columns()[0].dataType()); } @@ -1439,7 +1580,7 @@ void testOperationTableIndex() { Column[] newColumns = new Column[] {col1, col2, col3}; TableCatalog tableCatalog = catalog.asTableCatalog(); tableCatalog.createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), newColumns, table_comment, createProperties(), @@ -1450,7 +1591,7 @@ void testOperationTableIndex() { // add index test. tableCatalog.alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.addIndex( Index.IndexType.UNIQUE_KEY, "u1_key", new String[][] {{"col_2"}, {"col_3"}}), TableChange.addIndex( @@ -1458,8 +1599,7 @@ void testOperationTableIndex() { Indexes.DEFAULT_MYSQL_PRIMARY_KEY_NAME, new String[][] {{"col_1"}})); - Table table = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + Table table = tableCatalog.loadTable(NameIdentifier.of(schemaName, tableName)); Index[] indexes = new Index[] { Indexes.unique("u1_key", new String[][] {{"col_2"}, {"col_3"}}), @@ -1470,7 +1610,7 @@ void testOperationTableIndex() { // delete index and add new column and index. tableCatalog.alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.deleteIndex("u1_key", false), TableChange.addColumn( new String[] {"col_4"}, @@ -1483,8 +1623,7 @@ void testOperationTableIndex() { Indexes.createMysqlPrimaryKey(new String[][] {{"col_1"}}), Indexes.unique("u2_key", new String[][] {{"col_4"}}) }; - table = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + table = tableCatalog.loadTable(NameIdentifier.of(schemaName, tableName)); Column col4 = Column.of("col_4", Types.VarCharType.of(255), null, true, false, null); newColumns = new Column[] {col1, col2, col3, col4}; ITUtils.assertionsTableInfo( @@ -1492,7 +1631,7 @@ void testOperationTableIndex() { // Add a previously existing index tableCatalog.alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.addIndex( Index.IndexType.UNIQUE_KEY, "u1_key", new String[][] {{"col_2"}, {"col_3"}}), TableChange.addIndex( @@ -1505,8 +1644,7 @@ void testOperationTableIndex() { Indexes.unique("u1_key", new String[][] {{"col_2"}, {"col_3"}}), Indexes.unique("u3_key", new String[][] {{"col_1"}, {"col_4"}}) }; - table = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + table = tableCatalog.loadTable(NameIdentifier.of(schemaName, tableName)); ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), createProperties(), indexes, table); } @@ -1521,8 +1659,7 @@ void testAddColumnAutoIncrement() { String tableName = "auto_increment_table"; Column[] newColumns = new Column[] {col1, col2, col3, col4, col5}; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Map properties = createProperties(); TableCatalog tableCatalog = catalog.asTableCatalog(); tableCatalog.createTable( @@ -1606,8 +1743,7 @@ void testAddColumnDefaultValue() { String tableName = "default_value_table"; Column[] newColumns = new Column[] {col1, col2, col3}; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Map properties = createProperties(); TableCatalog tableCatalog = catalog.asTableCatalog(); tableCatalog.createTable( @@ -1644,4 +1780,48 @@ void testAddColumnDefaultValue() { Indexes.EMPTY_INDEXES, table); } + + @Test + public void testMySqlIntegerTypes() { + Column col1 = Column.of("col_1", Types.ByteType.get(), "byte type", true, false, null); + Column col2 = + Column.of("col_2", Types.ByteType.unsigned(), "byte unsigned type", true, false, null); + Column col3 = Column.of("col_3", Types.ShortType.get(), "short type", true, false, null); + Column col4 = + Column.of("col_4", Types.ShortType.unsigned(), "short unsigned type ", true, false, null); + Column col5 = Column.of("col_5", Types.IntegerType.get(), "integer type", true, false, null); + Column col6 = + Column.of( + "col_6", Types.IntegerType.unsigned(), "integer unsigned type", true, false, null); + Column col7 = Column.of("col_7", Types.LongType.get(), "long type", true, false, null); + Column col8 = + Column.of("col_8", Types.LongType.unsigned(), "long unsigned type", true, false, null); + String tableName = "default_integer_types_table"; + Column[] newColumns = new Column[] {col1, col2, col3, col4, col5, col6, col7, col8}; + + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); + Map properties = createProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + tableCatalog.createTable( + tableIdentifier, + newColumns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + Indexes.EMPTY_INDEXES); + + Table table = tableCatalog.loadTable(tableIdentifier); + Assertions.assertEquals(8, table.columns().length); + Column[] columns = table.columns(); + Assertions.assertEquals(columns[0].dataType().simpleString(), "byte"); + Assertions.assertEquals(columns[1].dataType().simpleString(), "byte unsigned"); + Assertions.assertEquals(columns[2].dataType().simpleString(), "short"); + Assertions.assertEquals(columns[3].dataType().simpleString(), "short unsigned"); + Assertions.assertEquals(columns[4].dataType().simpleString(), "integer"); + Assertions.assertEquals(columns[5].dataType().simpleString(), "integer unsigned"); + Assertions.assertEquals(columns[6].dataType().simpleString(), "long"); + Assertions.assertEquals(columns[7].dataType().simpleString(), "long unsigned"); + } } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlVersion5IT.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlVersion5IT.java index fb8ee3ae141..3e582d656ba 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlVersion5IT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlVersion5IT.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.integration.test; @@ -19,7 +32,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class CatalogMysqlVersion5IT extends CatalogMysqlIT { public CatalogMysqlVersion5IT() { super(); @@ -65,11 +78,7 @@ void testColumnDefaultValue() { catalog .asTableCatalog() .createTable( - NameIdentifier.of( - metalakeName, - catalogName, - schemaName, - GravitinoITUtils.genRandomName("mysql_it_table")), + NameIdentifier.of(schemaName, GravitinoITUtils.genRandomName("mysql_it_table")), newColumns, null, ImmutableMap.of()); diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/service/MysqlService.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/service/MysqlService.java index 228bdb21494..656926bda60 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/service/MysqlService.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/service/MysqlService.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.integration.test.service; diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/operation/TestMysql.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/operation/TestMysql.java index 7db171d5f92..64f3954f202 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/operation/TestMysql.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/operation/TestMysql.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.operation; diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/operation/TestMysqlDatabaseOperations.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/operation/TestMysqlDatabaseOperations.java index 2cf3d6bc14e..4cf92d3e349 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/operation/TestMysqlDatabaseOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/operation/TestMysqlDatabaseOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.operation; @@ -21,7 +35,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class TestMysqlDatabaseOperations extends TestMysql { @Test diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java index 9a38960e7ce..78c84f9cc89 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.mysql.operation; @@ -34,7 +48,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class TestMysqlTableOperations extends TestMysql { private static Type VARCHAR = Types.VarCharType.of(255); private static Type INT = Types.IntegerType.get(); diff --git a/catalogs/catalog-jdbc-mysql/src/test/resources/log4j2.properties b/catalogs/catalog-jdbc-mysql/src/test/resources/log4j2.properties index 9e5294bd1c6..a6dfcb14e8d 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/resources/log4j2.properties +++ b/catalogs/catalog-jdbc-mysql/src/test/resources/log4j2.properties @@ -1,6 +1,20 @@ # -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # Set to debug or trace if log4j initialization is failing diff --git a/catalogs/catalog-jdbc-postgresql/build.gradle.kts b/catalogs/catalog-jdbc-postgresql/build.gradle.kts index 2b545549dfb..af89c2dbf32 100644 --- a/catalogs/catalog-jdbc-postgresql/build.gradle.kts +++ b/catalogs/catalog-jdbc-postgresql/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ description = "catalog-jdbc-postgresql" diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSQLCatalogOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSQLCatalogOperations.java index 79f0314429f..399fe1bc4d7 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSQLCatalogOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSQLCatalogOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql; diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSqlCatalog.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSqlCatalog.java index 90d7ad1c16f..74985cc7424 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSqlCatalog.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSqlCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql; @@ -16,6 +30,7 @@ import com.datastrato.gravitino.catalog.postgresql.operation.PostgreSqlSchemaOperations; import com.datastrato.gravitino.catalog.postgresql.operation.PostgreSqlTableOperations; import com.datastrato.gravitino.connector.CatalogOperations; +import com.datastrato.gravitino.connector.capability.Capability; import java.util.Map; public class PostgreSqlCatalog extends JdbcCatalog { @@ -27,7 +42,7 @@ public String shortName() { @Override protected CatalogOperations newOps(Map config) { - JdbcTypeConverter jdbcTypeConverter = createJdbcTypeConverter(); + JdbcTypeConverter jdbcTypeConverter = createJdbcTypeConverter(); return new PostgreSQLCatalogOperations( createExceptionConverter(), jdbcTypeConverter, @@ -36,6 +51,11 @@ protected CatalogOperations newOps(Map config) { createJdbcColumnDefaultValueConverter()); } + @Override + public Capability newCapability() { + return new PostgreSqlCatalogCapability(); + } + @Override protected JdbcExceptionConverter createExceptionConverter() { return new PostgreSqlExceptionConverter(); diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSqlCatalogCapability.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSqlCatalogCapability.java new file mode 100644 index 00000000000..7dc63d4ab07 --- /dev/null +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/PostgreSqlCatalogCapability.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.postgresql; + +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.connector.capability.CapabilityResult; + +public class PostgreSqlCatalogCapability implements Capability { + /** + * Regular expression explanation: ^[_a-zA-Z\p{L}/][\w\p{L}-$/=]{0,62}$ + * + *

^[_a-zA-Z\p{L}/] - Start with an underscore, a letter, or a letter from any language + * + *

[\w\p{L}-$/=]{0,62} - Consist of 0 to 62 characters (making the total length at most 63) of + * letters (both cases), digits, underscores, any kind of letter from any language, hyphens, + * dollar signs, slashes or equal signs + * + *

$ - End of the string + */ + public static final String POSTGRESQL_NAME_PATTERN = "^[_a-zA-Z\\p{L}/][\\w\\p{L}-$/=]{0,62}$"; + + @Override + public CapabilityResult specificationOnName(Scope scope, String name) { + // TODO: Validate the name against reserved words + if (!name.matches(POSTGRESQL_NAME_PATTERN)) { + return CapabilityResult.unsupported( + String.format("The %s name '%s' is illegal.", scope, name)); + } + return CapabilityResult.SUPPORTED; + } +} diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlColumnDefaultValueConverter.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlColumnDefaultValueConverter.java index 27326770ff2..26211573e77 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlColumnDefaultValueConverter.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlColumnDefaultValueConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.converter; diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlExceptionConverter.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlExceptionConverter.java index bec19f6bc6a..fffef769e28 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlExceptionConverter.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlExceptionConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.converter; diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java index 77956e171e5..ef468c4f3a4 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.converter; @@ -11,7 +25,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -public class PostgreSqlTypeConverter extends JdbcTypeConverter { +public class PostgreSqlTypeConverter extends JdbcTypeConverter { static final String BOOL = "bool"; static final String INT_2 = "int2"; @@ -28,7 +42,7 @@ public class PostgreSqlTypeConverter extends JdbcTypeConverter { @VisibleForTesting static final String ARRAY_TOKEN = "[]"; @Override - public Type toGravitinoType(JdbcTypeBean typeBean) { + public Type toGravitino(JdbcTypeBean typeBean) { String typeName = typeBean.getTypeName().toLowerCase(); if (typeName.startsWith(JDBC_ARRAY_PREFIX)) { return toGravitinoArrayType(typeName); @@ -71,7 +85,7 @@ public Type toGravitinoType(JdbcTypeBean typeBean) { } @Override - public String fromGravitinoType(Type type) { + public String fromGravitino(Type type) { if (type instanceof Types.BooleanType) { return BOOL; } else if (type instanceof Types.ShortType) { @@ -130,13 +144,13 @@ private String fromGravitinoArrayType(ListType listType) { Preconditions.checkArgument( !(elementType instanceof ListType), "PostgreSQL doesn't support multidimensional list internally, please use one dimensional list"); - String elementTypeString = fromGravitinoType(elementType); + String elementTypeString = fromGravitino(elementType); return elementTypeString + ARRAY_TOKEN; } private ListType toGravitinoArrayType(String typeName) { String elementTypeName = typeName.substring(JDBC_ARRAY_PREFIX.length(), typeName.length()); JdbcTypeBean bean = new JdbcTypeBean(elementTypeName); - return ListType.of(toGravitinoType(bean), false); + return ListType.of(toGravitino(bean), false); } } diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java index 978121434b7..91f7071babf 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.operation; diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java index 57e9006655f..f581b86cbb2 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.operation; @@ -210,10 +224,7 @@ private static String getIndexFieldStr(String[][] fieldNames) { private void appendColumnDefinition(JdbcColumn column, StringBuilder sqlBuilder) { // Add data type - sqlBuilder - .append(SPACE) - .append(typeConverter.fromGravitinoType(column.dataType())) - .append(SPACE); + sqlBuilder.append(SPACE).append(typeConverter.fromGravitino(column.dataType())).append(SPACE); if (column.autoIncrement()) { if (!Types.allowAutoIncrement(column.dataType())) { @@ -532,7 +543,7 @@ private String updateColumnTypeFieldDefinition( .append(col) .append(PG_QUOTE) .append(" SET DATA TYPE ") - .append(typeConverter.fromGravitinoType(updateColumnType.getNewDataType())); + .append(typeConverter.fromGravitino(updateColumnType.getNewDataType())); if (!column.nullable()) { sqlBuilder .append(",\n") @@ -591,7 +602,7 @@ private List addColumnFieldDefinition( .append(col) .append(PG_QUOTE) .append(SPACE) - .append(typeConverter.fromGravitinoType(addColumn.getDataType())) + .append(typeConverter.fromGravitino(addColumn.getDataType())) .append(SPACE); if (addColumn.isAutoIncrement()) { diff --git a/catalogs/catalog-jdbc-postgresql/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider b/catalogs/catalog-jdbc-postgresql/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider index 8c5a726340c..f3a2b68e26a 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider +++ b/catalogs/catalog-jdbc-postgresql/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider @@ -1,5 +1,19 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # com.datastrato.gravitino.catalog.postgresql.PostgreSqlCatalog \ No newline at end of file diff --git a/catalogs/catalog-jdbc-postgresql/src/main/resources/jdbc-postgresql.conf b/catalogs/catalog-jdbc-postgresql/src/main/resources/jdbc-postgresql.conf index 2e2918753c7..94a9ce8eaac 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/resources/jdbc-postgresql.conf +++ b/catalogs/catalog-jdbc-postgresql/src/main/resources/jdbc-postgresql.conf @@ -1,6 +1,20 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # jdbc-url = jdbc:postgresql://localhost:5432/your_database # jdbc-user = strato diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java index 4710073d300..3ef38f4d6c4 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.converter; @@ -94,19 +108,17 @@ public void testFromGravitinoType() { checkGravitinoTypeToJdbcType(USER_DEFINED_TYPE, Types.ExternalType.of(USER_DEFINED_TYPE)); Assertions.assertThrows( IllegalArgumentException.class, - () -> - POSTGRE_SQL_TYPE_CONVERTER.fromGravitinoType(Types.UnparsedType.of(USER_DEFINED_TYPE))); + () -> POSTGRE_SQL_TYPE_CONVERTER.fromGravitino(Types.UnparsedType.of(USER_DEFINED_TYPE))); } protected void checkGravitinoTypeToJdbcType(String jdbcTypeName, Type gravitinoType) { - Assertions.assertEquals( - jdbcTypeName, POSTGRE_SQL_TYPE_CONVERTER.fromGravitinoType(gravitinoType)); + Assertions.assertEquals(jdbcTypeName, POSTGRE_SQL_TYPE_CONVERTER.fromGravitino(gravitinoType)); } protected void checkJdbcTypeToGravitinoType( Type gravitinoType, String jdbcTypeName, String columnSize, String scale) { JdbcTypeConverter.JdbcTypeBean typeBean = createTypeBean(jdbcTypeName, columnSize, scale); - Assertions.assertEquals(gravitinoType, POSTGRE_SQL_TYPE_CONVERTER.toGravitinoType(typeBean)); + Assertions.assertEquals(gravitinoType, POSTGRE_SQL_TYPE_CONVERTER.toGravitino(typeBean)); } protected static JdbcTypeConverter.JdbcTypeBean createTypeBean( diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java index f369ba518d9..0b0add476a9 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.integration.test; @@ -42,6 +56,7 @@ import com.datastrato.gravitino.rel.types.Decimal; import com.datastrato.gravitino.rel.types.Types; import com.datastrato.gravitino.rel.types.Types.IntegerType; +import com.datastrato.gravitino.utils.RandomNameUtils; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -65,7 +80,7 @@ import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) public class CatalogPostgreSqlIT extends AbstractIT { private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); @@ -126,7 +141,7 @@ public void resetSchema() { private void clearTableAndSchema() { NameIdentifier[] nameIdentifiers = - catalog.asTableCatalog().listTables(Namespace.of(metalakeName, catalogName, schemaName)); + catalog.asTableCatalog().listTables(Namespace.of(schemaName)); for (NameIdentifier nameIdentifier : nameIdentifiers) { catalog.asTableCatalog().dropTable(nameIdentifier); } @@ -166,11 +181,10 @@ private void createCatalog() throws SQLException { } private void createSchema() { - NameIdentifier ident = NameIdentifier.of(metalakeName, catalogName, schemaName); Schema createdSchema = - catalog.asSchemas().createSchema(ident.name(), schema_comment, Collections.EMPTY_MAP); - Schema loadSchema = catalog.asSchemas().loadSchema(ident.name()); + catalog.asSchemas().createSchema(schemaName, schema_comment, Collections.EMPTY_MAP); + Schema loadSchema = catalog.asSchemas().loadSchema(schemaName); Assertions.assertEquals(createdSchema.name(), loadSchema.name()); Assertions.assertEquals(createdSchema.comment(), loadSchema.comment()); } @@ -230,8 +244,7 @@ void testCreateTableWithArrayType() { Column col = Column.of("array", Types.ListType.of(IntegerType.get(), false), "col_4_comment"); Column[] columns = new Column[] {col}; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); TableCatalog tableCatalog = catalog.asTableCatalog(); Table createdTable = @@ -256,8 +269,7 @@ void testCreateTableWithSpecialColumnNames() { // Create table from Gravitino API Column[] columns = columnsWithSpecialNames(); - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Distribution distribution = Distributions.NONE; SortOrder[] sortOrders = new SortOrder[0]; @@ -302,7 +314,7 @@ void testOperationPostgreSqlSchema() { // create schema check. String testSchemaName = GravitinoITUtils.genRandomName("test_schema_1"); NameIdentifier schemaIdent = NameIdentifier.of(metalakeName, catalogName, testSchemaName); - schemas.createSchema(schemaIdent.name(), schema_comment, Collections.emptyMap()); + schemas.createSchema(testSchemaName, schema_comment, Collections.emptyMap()); nameIdentifiers = schemas.listSchemas(); schemaNames = Sets.newHashSet(nameIdentifiers); Assertions.assertTrue(schemaNames.contains(testSchemaName)); @@ -316,13 +328,12 @@ void testOperationPostgreSqlSchema() { Assertions.assertThrows( SchemaAlreadyExistsException.class, () -> { - schemas.createSchema(schemaIdent.name(), schema_comment, emptyMap); + schemas.createSchema(testSchemaName, schema_comment, emptyMap); }); // drop schema check. - schemas.dropSchema(schemaIdent.name(), false); - Assertions.assertThrows( - NoSuchSchemaException.class, () -> schemas.loadSchema(schemaIdent.name())); + schemas.dropSchema(testSchemaName, false); + Assertions.assertThrows(NoSuchSchemaException.class, () -> schemas.loadSchema(testSchemaName)); Assertions.assertThrows( NoSuchSchemaException.class, () -> postgreSqlService.loadSchema(schemaIdent)); @@ -333,8 +344,7 @@ void testOperationPostgreSqlSchema() { TableCatalog tableCatalog = catalog.asTableCatalog(); // create failed check. - NameIdentifier table = - NameIdentifier.of(metalakeName, catalogName, testSchemaName, "test_table"); + NameIdentifier table = NameIdentifier.of(testSchemaName, "test_table"); Assertions.assertThrows( NoSuchSchemaException.class, () -> @@ -356,13 +366,82 @@ void testOperationPostgreSqlSchema() { Assertions.assertTrue(schemaNames.contains(schemaName)); } + @Test + void testSchemaWithIllegalName() { + SupportsSchemas schemas = catalog.asSchemas(); + String schemaName = RandomNameUtils.genRandomName("ct_db"); + + // should throw an exception with string that might contain SQL injection + String sqlInjection = schemaName + "; DROP TABLE important_table; -- "; + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.createSchema(sqlInjection, null, null); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.dropSchema(sqlInjection, false); + }); + + String sqlInjection1 = schemaName + "; SELECT pg_sleep(10);"; + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.createSchema(sqlInjection1, null, null); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.dropSchema(sqlInjection1, false); + }); + + String sqlInjection2 = + schemaName + "`; UPDATE Users SET password = 'newpassword' WHERE username = 'admin'; -- "; + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.createSchema(sqlInjection2, null, null); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.dropSchema(sqlInjection2, false); + }); + + // should throw an exception with input that has more than 63 characters + String invalidInput = StringUtils.repeat("a", 64); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.createSchema(invalidInput, null, null); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.dropSchema(invalidInput, false); + }); + + // should throw an exception with schema name that starts with special character + String invalidInput2 = RandomNameUtils.genRandomName("$test_db"); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.createSchema(invalidInput2, null, null); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + schemas.dropSchema(invalidInput2, false); + }); + } + @Test void testCreateAndLoadPostgreSqlTable() { // Create table from Gravitino API Column[] columns = createColumns(); - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Distribution distribution = Distributions.NONE; SortOrder[] sortOrders = new SortOrder[0]; @@ -412,7 +491,7 @@ void testAlterAndDropPostgreSqlTable() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), columns, table_comment, createProperties()); @@ -424,7 +503,7 @@ void testAlterAndDropPostgreSqlTable() { catalog .asTableCatalog() .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.rename(alertTableName), TableChange.updateComment(table_comment + "_new")); }); @@ -434,8 +513,7 @@ void testAlterAndDropPostgreSqlTable() { catalog .asTableCatalog() .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), - TableChange.rename(alertTableName)); + NameIdentifier.of(schemaName, tableName), TableChange.rename(alertTableName)); Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().creator()); Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().lastModifier()); @@ -443,17 +521,14 @@ void testAlterAndDropPostgreSqlTable() { catalog .asTableCatalog() .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, alertTableName), + NameIdentifier.of(schemaName, alertTableName), TableChange.updateComment(table_comment + "_new"), TableChange.addColumn(new String[] {"col_4"}, Types.StringType.get()), TableChange.renameColumn(new String[] {POSTGRESQL_COL_NAME2}, "col_2_new"), TableChange.updateColumnType( new String[] {POSTGRESQL_COL_NAME1}, Types.IntegerType.get())); - table = - catalog - .asTableCatalog() - .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, alertTableName)); + table = catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName, alertTableName)); Assertions.assertEquals(alertTableName, table.name()); Assertions.assertEquals(POSTGRESQL_COL_NAME1, table.columns()[0].name()); @@ -486,8 +561,7 @@ void testAlterAndDropPostgreSqlTable() { Column[] newColumns = new Column[] {col1, col2, col3}; NameIdentifier tableIdentifier = - NameIdentifier.of( - metalakeName, catalogName, schemaName, GravitinoITUtils.genRandomName("jdbc_it_table")); + NameIdentifier.of(schemaName, GravitinoITUtils.genRandomName("jdbc_it_table")); catalog .asTableCatalog() .createTable( @@ -556,8 +630,7 @@ public void testBackQuoteTable() { Column col4 = Column.of("status", Types.VarCharType.of(255), "code", false, false, null); Column[] newColumns = new Column[] {col1, col2, col3, col4}; TableCatalog tableCatalog = catalog.asTableCatalog(); - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, "abc"); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, "abc"); Assertions.assertDoesNotThrow( () -> tableCatalog.createTable( @@ -609,8 +682,7 @@ void testCreateIndexTable() { Indexes.unique("u6_key", new String[][] {{"col_1"}, {"col_3"}, {"col_2"}, {"col_4"}}), }; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); // Test create many indexes with name success. Map properties = createProperties(); @@ -632,7 +704,7 @@ void testCreateIndexTable() { tableName, table_comment, Arrays.asList(newColumns), properties, indexes, table); // Test create index complex fields fail. - NameIdentifier id = NameIdentifier.of(metalakeName, catalogName, schemaName, "test_failed"); + NameIdentifier id = NameIdentifier.of(schemaName, "test_failed"); SortOrder[] sortOrder = new SortOrder[0]; Index[] primaryIndex = new Index[] {Indexes.createMysqlPrimaryKey(new String[][] {{"col_1", "col_2"}})}; @@ -679,7 +751,7 @@ void testCreateIndexTable() { // Test create index with empty name success. table = tableCatalog.createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, "test_null_key"), + NameIdentifier.of(schemaName, "test_null_key"), newColumns, table_comment, properties, @@ -700,7 +772,7 @@ void testCreateIndexTable() { // Test create index with same col success. table = tableCatalog.createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, "many_index"), + NameIdentifier.of(schemaName, "many_index"), newColumns, table_comment, properties, @@ -759,11 +831,7 @@ void testColumnDefaultValue() { catalog .asTableCatalog() .createTable( - NameIdentifier.of( - metalakeName, - catalogName, - schemaName, - GravitinoITUtils.genRandomName("pg_it_table")), + NameIdentifier.of(schemaName, GravitinoITUtils.genRandomName("pg_it_table")), newColumns, null, ImmutableMap.of()); @@ -788,7 +856,7 @@ void testUpdateColumnDefaultValue() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), columns, table_comment, createProperties()); @@ -797,7 +865,7 @@ void testUpdateColumnDefaultValue() { catalog .asTableCatalog() .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.updateColumnDefaultValue( new String[] {columns[0].name()}, Literals.of("1.234", Types.FloatType.get())), TableChange.updateColumnDefaultValue( @@ -809,10 +877,7 @@ void testUpdateColumnDefaultValue() { TableChange.updateColumnDefaultValue( new String[] {columns[4].name()}, FunctionExpression.of("current_timestamp"))); - table = - catalog - .asTableCatalog() - .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + table = catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName, tableName)); Assertions.assertEquals( Literals.of("1.234", Types.FloatType.get()), table.columns()[0].defaultValue()); @@ -856,9 +921,7 @@ void testColumnDefaultValueConverter() { System.out.println(sql); postgreSqlService.executeQuery(sql); Table loadedTable = - catalog - .asTableCatalog() - .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName, tableName)); for (Column column : loadedTable.columns()) { switch (column.name()) { @@ -937,8 +1000,7 @@ void testPGSpecialTableName() { Index[] t1_indexes = {Indexes.unique("u1_key", new String[][] {{t1_name}})}; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, t1_name); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, t1_name); tableCatalog.createTable( tableIdentifier, columns, @@ -953,7 +1015,7 @@ void testPGSpecialTableName() { Column t2_col = Column.of(t2_name, Types.LongType.get(), "id", false, false, null); Index[] t2_indexes = {Indexes.unique("u2_key", new String[][] {{t2_name}})}; columns = new Column[] {t2_col}; - tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, t2_name); + tableIdentifier = NameIdentifier.of(schemaName, t2_name); tableCatalog.createTable( tableIdentifier, columns, @@ -968,7 +1030,7 @@ void testPGSpecialTableName() { Column t3_col = Column.of(t3_name, Types.LongType.get(), "id", false, false, null); Index[] t3_indexes = {Indexes.unique("u3_key", new String[][] {{t3_name}})}; columns = new Column[] {t3_col}; - tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, t3_name); + tableIdentifier = NameIdentifier.of(schemaName, t3_name); tableCatalog.createTable( tableIdentifier, columns, @@ -983,7 +1045,7 @@ void testPGSpecialTableName() { Column t4_col = Column.of(t4_name, Types.LongType.get(), "id", false, false, null); Index[] t4_indexes = {Indexes.unique("u4_key", new String[][] {{t4_name}})}; columns = new Column[] {t4_col}; - tableIdentifier = NameIdentifier.of(metalakeName, catalogName, schemaName, t4_name); + tableIdentifier = NameIdentifier.of(schemaName, t4_name); tableCatalog.createTable( tableIdentifier, columns, @@ -994,31 +1056,165 @@ void testPGSpecialTableName() { new SortOrder[0], t4_indexes); - Table t1 = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t1_name)); + Table t1 = tableCatalog.loadTable(NameIdentifier.of(schemaName, t1_name)); Arrays.stream(t1.columns()).anyMatch(c -> Objects.equals(c.name(), "t112")); ITUtils.assertionsTableInfo( t1_name, table_comment, Arrays.asList(t1_col), properties, t1_indexes, t1); - Table t2 = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t2_name)); + Table t2 = tableCatalog.loadTable(NameIdentifier.of(schemaName, t2_name)); Arrays.stream(t2.columns()).anyMatch(c -> Objects.equals(c.name(), "t212")); ITUtils.assertionsTableInfo( t2_name, table_comment, Arrays.asList(t2_col), properties, t2_indexes, t2); - Table t3 = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t3_name)); + Table t3 = tableCatalog.loadTable(NameIdentifier.of(schemaName, t3_name)); Arrays.stream(t3.columns()).anyMatch(c -> Objects.equals(c.name(), "t_12")); ITUtils.assertionsTableInfo( t3_name, table_comment, Arrays.asList(t3_col), properties, t3_indexes, t3); - Table t4 = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, t4_name)); + Table t4 = tableCatalog.loadTable(NameIdentifier.of(schemaName, t4_name)); Arrays.stream(t4.columns()).anyMatch(c -> Objects.equals(c.name(), "_1__")); ITUtils.assertionsTableInfo( t4_name, table_comment, Arrays.asList(t4_col), properties, t4_indexes, t4); } + @Test + void testPGIllegalTableName() { + Map properties = createProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + String table_name = "t123"; + + String t1_name = table_name + "`; DROP TABLE important_table; -- "; + Column t1_col = Column.of(t1_name, Types.LongType.get(), "id", false, false, null); + Column[] columns = {t1_col}; + Index[] t1_indexes = {Indexes.unique("u1_key", new String[][] {{t1_name}})}; + NameIdentifier tableIdentifier = + NameIdentifier.of(metalakeName, catalogName, schemaName, t1_name); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t1_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier); + }); + + String t2_name = table_name + "`; SLEEP(10); -- "; + Column t2_col = Column.of(t2_name, Types.LongType.get(), "id", false, false, null); + Index[] t2_indexes = {Indexes.unique("u2_key", new String[][] {{t2_name}})}; + Column[] columns2 = new Column[] {t2_col}; + NameIdentifier tableIdentifier2 = + NameIdentifier.of(metalakeName, catalogName, schemaName, t2_name); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier2, + columns2, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t2_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier2); + }); + + String t3_name = + table_name + "`; UPDATE Users SET password = 'newpassword' WHERE username = 'admin'; -- "; + Column t3_col = Column.of(t3_name, Types.LongType.get(), "id", false, false, null); + Index[] t3_indexes = {Indexes.unique("u3_key", new String[][] {{t3_name}})}; + Column[] columns3 = new Column[] {t3_col}; + NameIdentifier tableIdentifier3 = + NameIdentifier.of(metalakeName, catalogName, schemaName, t3_name); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier3, + columns3, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t3_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier3); + }); + + String invalidInput = StringUtils.repeat("a", 64); + Column t4_col = Column.of(invalidInput, Types.LongType.get(), "id", false, false, null); + Index[] t4_indexes = {Indexes.unique("u4_key", new String[][] {{invalidInput}})}; + Column[] columns4 = new Column[] {t4_col}; + NameIdentifier tableIdentifier4 = + NameIdentifier.of(metalakeName, catalogName, schemaName, invalidInput); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier4, + columns4, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t4_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier4); + }); + + String invalidInput2 = RandomNameUtils.genRandomName("$test_db"); + Column t5_col = Column.of(invalidInput2, Types.LongType.get(), "id", false, false, null); + Index[] t5_indexes = {Indexes.unique("u5_key", new String[][] {{invalidInput2}})}; + Column[] columns5 = new Column[] {t5_col}; + NameIdentifier tableIdentifier5 = + NameIdentifier.of(metalakeName, catalogName, schemaName, invalidInput2); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + tableIdentifier5, + columns5, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + t5_indexes); + }); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + catalog.asTableCatalog().dropTable(tableIdentifier5); + }); + } + @Test void testPGTableNameCaseSensitive() { Column col1 = Column.of("col_1", Types.LongType.get(), "id", false, false, null); @@ -1029,8 +1225,7 @@ void testPGTableNameCaseSensitive() { Column[] newColumns = new Column[] {col1, col2, col3, col4, col5}; Index[] indexes = new Index[0]; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, "tablename"); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, "tablename"); Map properties = createProperties(); TableCatalog tableCatalog = catalog.asTableCatalog(); Table createdTable = @@ -1055,8 +1250,7 @@ void testPGTableNameCaseSensitive() { "tablename", "low case table name", Arrays.asList(newColumns), properties, indexes, table); // Test create table with same name but different case - NameIdentifier tableIdentifier2 = - NameIdentifier.of(metalakeName, catalogName, schemaName, "TABLENAME"); + NameIdentifier tableIdentifier2 = NameIdentifier.of(schemaName, "TABLENAME"); Column[] upperTableColumns = new Column[] {col1, col4, col5}; Table tableAgain = @@ -1112,7 +1306,7 @@ void testPGListTable() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, dbs[i], tables[i]), + NameIdentifier.of(dbs[i], tables[i]), newColumns, dbs[i] + "." + tables[i], Maps.newHashMap(), @@ -1124,8 +1318,7 @@ void testPGListTable() { // list table in schema1 for (int i = 0; i < 5; i++) { - NameIdentifier[] tableNames = - catalog.asTableCatalog().listTables(Namespace.of(metalakeName, catalogName, dbs[i])); + NameIdentifier[] tableNames = catalog.asTableCatalog().listTables(Namespace.of(dbs[i])); Assertions.assertEquals(1, tableNames.length); Assertions.assertEquals(tables[i], tableNames[0].name()); } @@ -1160,7 +1353,7 @@ void testCreateSameTableInDifferentSchema() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, dbs[i], tables[j]), + NameIdentifier.of(dbs[i], tables[j]), newColumns, dbs[i] + "." + tables[j], Maps.newHashMap(), @@ -1173,8 +1366,7 @@ void testCreateSameTableInDifferentSchema() { // list table in schema for (int i = 0; i < dbs.length; i++) { - NameIdentifier[] tableNames = - catalog.asTableCatalog().listTables(Namespace.of(metalakeName, catalogName, dbs[i])); + NameIdentifier[] tableNames = catalog.asTableCatalog().listTables(Namespace.of(dbs[i])); Assertions.assertEquals(3, tableNames.length); String[] realNames = Arrays.stream(tableNames).map(NameIdentifier::name).toArray(String[]::new); @@ -1184,10 +1376,7 @@ void testCreateSameTableInDifferentSchema() { for (String n : realNames) { Table t = Assertions.assertDoesNotThrow( - () -> - catalog - .asTableCatalog() - .loadTable(NameIdentifier.of(metalakeName, catalogName, dbs[idx], n))); + () -> catalog.asTableCatalog().loadTable(NameIdentifier.of(dbs[idx], n))); Assertions.assertEquals(n, t.name()); // Test the table1 is the `1a`.`table1` not `_a`.`table1` or `__`.`table1` @@ -1222,7 +1411,7 @@ void testPostgreSQLSchemaNameCaseSensitive() { for (String schema : schemas) { tableCatalog.createTable( - NameIdentifier.of(metalakeName, catalogName, schema, tableName), + NameIdentifier.of(schema, tableName), newColumns, table_comment, properties, @@ -1231,8 +1420,7 @@ void testPostgreSQLSchemaNameCaseSensitive() { new SortOrder[0], indexes); tableCatalog.createTable( - NameIdentifier.of( - metalakeName, catalogName, schema, GravitinoITUtils.genRandomName("test2")), + NameIdentifier.of(schema, GravitinoITUtils.genRandomName("test2")), newColumns, table_comment, properties, @@ -1243,8 +1431,7 @@ void testPostgreSQLSchemaNameCaseSensitive() { } for (String schema : schemas) { - NameIdentifier[] nameIdentifiers = - tableCatalog.listTables(Namespace.of(metalakeName, catalogName, schema)); + NameIdentifier[] nameIdentifiers = tableCatalog.listTables(Namespace.of(schema)); Assertions.assertEquals(2, nameIdentifiers.length); Assertions.assertTrue( Arrays.stream(nameIdentifiers) @@ -1261,9 +1448,7 @@ void testUnparsedTypeConverter() { postgreSqlService.executeQuery( String.format("CREATE TABLE %s.%s (bit_col bit);", schemaName, tableName)); Table loadedTable = - catalog - .asTableCatalog() - .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName, tableName)); Assertions.assertEquals(Types.ExternalType.of("bit"), loadedTable.columns()[0].dataType()); } @@ -1276,7 +1461,7 @@ void testOperationTableIndex() { Column[] newColumns = new Column[] {col1, col2, col3}; TableCatalog tableCatalog = catalog.asTableCatalog(); tableCatalog.createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), newColumns, table_comment, createProperties(), @@ -1287,13 +1472,12 @@ void testOperationTableIndex() { // add index test. tableCatalog.alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.addIndex( Index.IndexType.UNIQUE_KEY, "u1_key", new String[][] {{"col_2"}, {"col_3"}}), TableChange.addIndex(Index.IndexType.PRIMARY_KEY, "pk1_key", new String[][] {{"col_1"}})); - Table table = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + Table table = tableCatalog.loadTable(NameIdentifier.of(schemaName, tableName)); Index[] indexes = new Index[] { Indexes.unique("u1_key", new String[][] {{"col_2"}, {"col_3"}}), @@ -1304,7 +1488,7 @@ void testOperationTableIndex() { // delete index and add new column and index. tableCatalog.alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.deleteIndex("u1_key", true), TableChange.addColumn( new String[] {"col_4"}, @@ -1317,8 +1501,7 @@ void testOperationTableIndex() { Indexes.primary("pk1_key", new String[][] {{"col_1"}}), Indexes.unique("u2_key", new String[][] {{"col_4"}}) }; - table = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + table = tableCatalog.loadTable(NameIdentifier.of(schemaName, tableName)); Column col4 = Column.of("col_4", Types.VarCharType.of(255), null, true, false, null); newColumns = new Column[] {col1, col2, col3, col4}; ITUtils.assertionsTableInfo( @@ -1326,7 +1509,7 @@ void testOperationTableIndex() { // Add a previously existing index tableCatalog.alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.addIndex( Index.IndexType.UNIQUE_KEY, "u1_key", new String[][] {{"col_2"}, {"col_3"}}), TableChange.addIndex( @@ -1339,8 +1522,7 @@ void testOperationTableIndex() { Indexes.unique("u1_key", new String[][] {{"col_2"}, {"col_3"}}), Indexes.unique("u3_key", new String[][] {{"col_1"}, {"col_4"}}) }; - table = - tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + table = tableCatalog.loadTable(NameIdentifier.of(schemaName, tableName)); ITUtils.assertionsTableInfo( tableName, table_comment, Arrays.asList(newColumns), createProperties(), indexes, table); } @@ -1354,8 +1536,7 @@ void testAddColumnAutoIncrement() { Column[] newColumns = new Column[] {col1, col2, col3, col4}; String tableName = "auto_increment_table"; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Map properties = createProperties(); TableCatalog tableCatalog = catalog.asTableCatalog(); tableCatalog.createTable( @@ -1427,8 +1608,7 @@ void testAddColumnDefaultValue() { Assertions.assertEquals(Column.DEFAULT_VALUE_NOT_SET, newColumns[0].defaultValue()); - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Map properties = createProperties(); TableCatalog tableCatalog = catalog.asTableCatalog(); tableCatalog.createTable( diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion12IT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion12IT.java index bfd9347939d..1231abfb612 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion12IT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion12IT.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.integration.test; @@ -8,7 +21,7 @@ import com.datastrato.gravitino.integration.test.container.PGImageName; import org.junit.jupiter.api.Tag; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class CatalogPostgreSqlVersion12IT extends CatalogPostgreSqlIT { public CatalogPostgreSqlVersion12IT() { postgreImageName = PGImageName.VERSION_12; diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion14IT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion14IT.java index b7990d8a696..f6c20d86485 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion14IT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion14IT.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.integration.test; @@ -8,7 +21,7 @@ import com.datastrato.gravitino.integration.test.container.PGImageName; import org.junit.jupiter.api.Tag; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class CatalogPostgreSqlVersion14IT extends CatalogPostgreSqlIT { public CatalogPostgreSqlVersion14IT() { postgreImageName = PGImageName.VERSION_14; diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion15IT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion15IT.java index f53113d9ce2..49fee570b8f 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion15IT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion15IT.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.integration.test; @@ -8,7 +21,7 @@ import com.datastrato.gravitino.integration.test.container.PGImageName; import org.junit.jupiter.api.Tag; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class CatalogPostgreSqlVersion15IT extends CatalogPostgreSqlIT { public CatalogPostgreSqlVersion15IT() { postgreImageName = PGImageName.VERSION_15; diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion16IT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion16IT.java index 975afe3014c..5a11efdd30b 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion16IT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlVersion16IT.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.integration.test; @@ -8,7 +21,7 @@ import com.datastrato.gravitino.integration.test.container.PGImageName; import org.junit.jupiter.api.Tag; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class CatalogPostgreSqlVersion16IT extends CatalogPostgreSqlIT { public CatalogPostgreSqlVersion16IT() { postgreImageName = PGImageName.VERSION_16; diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java index 973ad10c9e9..1787ef3122b 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.integration.test; @@ -28,7 +42,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class TestMultipleJDBCLoad extends AbstractIT { private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static final TestDatabaseName TEST_DB_NAME = @@ -93,7 +107,7 @@ public void testCreateMultipleJdbc() throws URISyntaxException, SQLException { mysqlCatalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, mysqlCatalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), new Column[] {col1}, comment, Collections.emptyMap()); @@ -101,19 +115,14 @@ public void testCreateMultipleJdbc() throws URISyntaxException, SQLException { postgreSqlCatalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, postgreSqlCatalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), new Column[] {col1}, comment, Collections.emptyMap()); Assertions.assertTrue( - mysqlCatalog - .asTableCatalog() - .tableExists(NameIdentifier.of(metalakeName, mysqlCatalogName, schemaName, tableName))); + mysqlCatalog.asTableCatalog().tableExists(NameIdentifier.of(schemaName, tableName))); Assertions.assertTrue( - postgreSqlCatalog - .asTableCatalog() - .tableExists( - NameIdentifier.of(metalakeName, postgreSqlCatalogName, schemaName, tableName))); + postgreSqlCatalog.asTableCatalog().tableExists(NameIdentifier.of(schemaName, tableName))); } } diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/service/PostgreSqlService.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/service/PostgreSqlService.java index 02612dac5d0..dde97aa112e 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/service/PostgreSqlService.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/service/PostgreSqlService.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.integration.test.service; diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/operation/TestPostgreSql.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/operation/TestPostgreSql.java index 80a3415f32d..af17e9b27af 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/operation/TestPostgreSql.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/operation/TestPostgreSql.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.operation; diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/operation/TestPostgreSqlSchemaOperations.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/operation/TestPostgreSqlSchemaOperations.java index 33292a1ddad..696426e428d 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/operation/TestPostgreSqlSchemaOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/operation/TestPostgreSqlSchemaOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.operation; @@ -24,7 +38,7 @@ import org.junit.jupiter.api.Test; import org.testcontainers.shaded.com.google.common.collect.Maps; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class TestPostgreSqlSchemaOperations extends TestPostgreSql { @Test diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperations.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperations.java index ae2ee11f44f..68f8842b24c 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.postgresql.operation; @@ -35,7 +49,7 @@ import org.junit.jupiter.api.Test; import org.testcontainers.shaded.com.google.common.collect.Maps; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class TestPostgreSqlTableOperations extends TestPostgreSql { private static Type VARCHAR = Types.VarCharType.of(255); diff --git a/catalogs/catalog-jdbc-postgresql/src/test/resources/log4j2.properties b/catalogs/catalog-jdbc-postgresql/src/test/resources/log4j2.properties index 4b0ce16abb3..e4c1f765b1b 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/resources/log4j2.properties +++ b/catalogs/catalog-jdbc-postgresql/src/test/resources/log4j2.properties @@ -1,6 +1,20 @@ # -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # Set to debug or trace if log4j initialization is failing diff --git a/catalogs/catalog-kafka/build.gradle.kts b/catalogs/catalog-kafka/build.gradle.kts index a4234f654c1..7903834335a 100644 --- a/catalogs/catalog-kafka/build.gradle.kts +++ b/catalogs/catalog-kafka/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ description = "catalog-kafka" diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalog.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalog.java index 73d3880908b..d428e33b624 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalog.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.kafka; diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogCapability.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogCapability.java index be2141ce331..6fd0a7511e0 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogCapability.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogCapability.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.kafka; diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java index 5133365245b..8254d94cf52 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.kafka; @@ -38,6 +52,7 @@ import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.storage.IdGenerator; +import com.datastrato.gravitino.utils.NamespaceUtil; import com.datastrato.gravitino.utils.PrincipalUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -552,7 +567,7 @@ private void createDefaultSchemaIfNecessary() { SchemaEntity.builder() .withName(defaultSchemaIdent.name()) .withId(uid) - .withNamespace(Namespace.ofSchema(info.namespace().level(0), info.name())) + .withNamespace(NamespaceUtil.ofSchema(info.namespace().level(0), info.name())) .withComment("The default schema of Kafka catalog including all topics") .withProperties(properties) .withAuditInfo( diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogPropertiesMetadata.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogPropertiesMetadata.java index b54b4d93690..591be0b5144 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogPropertiesMetadata.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogPropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.kafka; diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaSchema.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaSchema.java index 1c1c409aaf0..f42d9cdcc29 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaSchema.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaSchema.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.kafka; diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaSchemaPropertiesMetadata.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaSchemaPropertiesMetadata.java index 5263324ca27..c2b28ca3bb0 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaSchemaPropertiesMetadata.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaSchemaPropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.kafka; diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaTopic.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaTopic.java index 2e43ccf9427..7cd66e880bd 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaTopic.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaTopic.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.kafka; diff --git a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaTopicPropertiesMetadata.java b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaTopicPropertiesMetadata.java index d6a6bfe405e..4b15fcd5f1b 100644 --- a/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaTopicPropertiesMetadata.java +++ b/catalogs/catalog-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaTopicPropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.kafka; diff --git a/catalogs/catalog-kafka/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider b/catalogs/catalog-kafka/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider index 25e98c8ee99..a3c373bc342 100644 --- a/catalogs/catalog-kafka/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider +++ b/catalogs/catalog-kafka/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider @@ -1,5 +1,19 @@ # -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # com.datastrato.gravitino.catalog.kafka.KafkaCatalog diff --git a/catalogs/catalog-kafka/src/main/resources/kafka.conf b/catalogs/catalog-kafka/src/main/resources/kafka.conf index 2e2467154d6..f272e1a58f7 100644 --- a/catalogs/catalog-kafka/src/main/resources/kafka.conf +++ b/catalogs/catalog-kafka/src/main/resources/kafka.conf @@ -1,6 +1,20 @@ # -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # bootstrap.servers = localhost:9092 diff --git a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java index e7b242ed13c..2d29b2f68e3 100644 --- a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java +++ b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java @@ -1,14 +1,28 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.kafka; import static com.datastrato.gravitino.Catalog.Type.MESSAGING; import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_KV_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; import static com.datastrato.gravitino.Configs.ENTITY_KV_STORE; import static com.datastrato.gravitino.Configs.ENTITY_STORE; -import static com.datastrato.gravitino.Configs.ENTRY_KV_ROCKSDB_BACKEND_PATH; import static com.datastrato.gravitino.Configs.STORE_DELETE_AFTER_TIME; import static com.datastrato.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME; import static com.datastrato.gravitino.StringIdentifier.ID_KEY; @@ -100,9 +114,9 @@ public static void setUp() { Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv"); Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE); Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto"); - Mockito.when(config.get(ENTRY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(ROCKS_DB_STORE_PATH); + Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(ROCKS_DB_STORE_PATH); - Assertions.assertEquals(ROCKS_DB_STORE_PATH, config.get(ENTRY_KV_ROCKSDB_BACKEND_PATH)); + Assertions.assertEquals(ROCKS_DB_STORE_PATH, config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)); Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); diff --git a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java index 3164c904796..bca6df6c897 100644 --- a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java +++ b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.kafka.integration.test; @@ -57,7 +71,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class CatalogKafkaIT extends AbstractIT { private static final Logger LOG = LoggerFactory.getLogger(CatalogKafkaIT.class); private static final ContainerSuite CONTAINER_SUITE = ContainerSuite.getInstance(); @@ -108,8 +122,8 @@ public static void shutdown() { })); Arrays.stream(metalake.listCatalogs()) .forEach( - (ident -> { - metalake.dropCatalog(ident.name()); + (catalogName -> { + metalake.dropCatalog(catalogName); })); client.dropMetalake(METALAKE_NAME); if (adminClient != null) { @@ -209,11 +223,7 @@ public void testCatalogException() { exception = Assertions.assertThrows( RuntimeException.class, - () -> - kafka - .asTopicCatalog() - .listTopics( - Namespace.ofTopic(METALAKE_NAME, catalogName2, DEFAULT_SCHEMA_NAME))); + () -> kafka.asTopicCatalog().listTopics(Namespace.of(DEFAULT_SCHEMA_NAME))); Assertions.assertTrue( exception .getMessage() @@ -275,15 +285,12 @@ public void testCreateAndListTopic() throws ExecutionException, InterruptedExcep catalog .asTopicCatalog() .createTopic( - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName), + NameIdentifier.of(DEFAULT_SCHEMA_NAME, topicName), "comment", null, Collections.emptyMap()); Topic loadedTopic = - catalog - .asTopicCatalog() - .loadTopic( - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName)); + catalog.asTopicCatalog().loadTopic(NameIdentifier.of(DEFAULT_SCHEMA_NAME, topicName)); Assertions.assertEquals(createdTopic, loadedTopic); assertTopicWithKafka(createdTopic); @@ -291,9 +298,7 @@ public void testCreateAndListTopic() throws ExecutionException, InterruptedExcep // test list topics NameIdentifier[] topics = - catalog - .asTopicCatalog() - .listTopics(Namespace.ofTopic(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME)); + catalog.asTopicCatalog().listTopics(Namespace.of(DEFAULT_SCHEMA_NAME)); Assertions.assertTrue(topics.length > 0); Assertions.assertTrue( ImmutableList.copyOf(topics).stream().anyMatch(topic -> topic.name().equals(topicName))); @@ -306,7 +311,7 @@ public void testAlterTopic() { catalog .asTopicCatalog() .createTopic( - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName), + NameIdentifier.of(DEFAULT_SCHEMA_NAME, topicName), "comment", null, ImmutableMap.of(TopicConfig.RETENTION_MS_CONFIG, "43200000")); @@ -322,15 +327,12 @@ public void testAlterTopic() { catalog .asTopicCatalog() .alterTopic( - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName), + NameIdentifier.of(DEFAULT_SCHEMA_NAME, topicName), TopicChange.updateComment("new comment"), TopicChange.setProperty(PARTITION_COUNT, "3"), TopicChange.removeProperty(TopicConfig.RETENTION_MS_CONFIG)); Topic loadedTopic = - catalog - .asTopicCatalog() - .loadTopic( - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName)); + catalog.asTopicCatalog().loadTopic(NameIdentifier.of(DEFAULT_SCHEMA_NAME, topicName)); Assertions.assertEquals(alteredTopic, loadedTopic); Assertions.assertEquals("new comment", alteredTopic.comment()); @@ -348,16 +350,13 @@ public void testDropTopic() throws ExecutionException, InterruptedException { catalog .asTopicCatalog() .createTopic( - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName), + NameIdentifier.of(DEFAULT_SCHEMA_NAME, topicName), "comment", null, Collections.emptyMap()); boolean dropped = - catalog - .asTopicCatalog() - .dropTopic( - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName)); + catalog.asTopicCatalog().dropTopic(NameIdentifier.of(DEFAULT_SCHEMA_NAME, topicName)); Assertions.assertTrue(dropped); // verify topic not exist in Kafka @@ -371,23 +370,17 @@ public void testDropTopic() throws ExecutionException, InterruptedException { catalog .asTopicCatalog() .createTopic( - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName1), + NameIdentifier.of(DEFAULT_SCHEMA_NAME, topicName1), "comment", null, Collections.emptyMap()); adminClient.deleteTopics(Collections.singleton(topicName1)).all().get(); boolean dropped1 = - catalog - .asTopicCatalog() - .dropTopic( - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName1)); + catalog.asTopicCatalog().dropTopic(NameIdentifier.of(DEFAULT_SCHEMA_NAME, topicName1)); Assertions.assertFalse(dropped1, "Should return false when dropping non-exist topic"); Assertions.assertFalse( - catalog - .asTopicCatalog() - .topicExists( - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, topicName1)), + catalog.asTopicCatalog().topicExists(NameIdentifier.of(DEFAULT_SCHEMA_NAME, topicName1)), "Topic should not exist after dropping"); } @@ -397,8 +390,7 @@ public void testNameSpec() throws ExecutionException, InterruptedException { String illegalName = "test.topic"; adminClient.createTopics(ImmutableList.of(new NewTopic(illegalName, 1, (short) 1))).all().get(); - NameIdentifier ident = - NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME, illegalName); + NameIdentifier ident = NameIdentifier.of(DEFAULT_SCHEMA_NAME, illegalName); IllegalArgumentException exception = Assertions.assertThrows( IllegalArgumentException.class, @@ -412,9 +404,7 @@ public void testNameSpec() throws ExecutionException, InterruptedException { Assertions.assertEquals(illegalName, loadedTopic.name()); NameIdentifier[] topics = - catalog - .asTopicCatalog() - .listTopics(Namespace.ofTopic(METALAKE_NAME, CATALOG_NAME, DEFAULT_SCHEMA_NAME)); + catalog.asTopicCatalog().listTopics(Namespace.of(DEFAULT_SCHEMA_NAME)); Assertions.assertTrue( Arrays.stream(topics).anyMatch(topic -> topic.name().equals(illegalName))); diff --git a/catalogs/catalog-kafka/src/test/resources/log4j2.properties b/catalogs/catalog-kafka/src/test/resources/log4j2.properties index afb89fe28f5..a565d07879b 100644 --- a/catalogs/catalog-kafka/src/test/resources/log4j2.properties +++ b/catalogs/catalog-kafka/src/test/resources/log4j2.properties @@ -1,6 +1,20 @@ # -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # Set to debug or trace if log4j initialization is failing diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts index 366f6a6b1de..5aa6f8cdea1 100644 --- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts +++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ description = "catalog-lakehouse-iceberg" @@ -25,6 +39,8 @@ dependencies { implementation(libs.bundles.jetty) implementation(libs.bundles.jersey) implementation(libs.bundles.log4j) + implementation(libs.caffeine) + implementation(libs.cglib) implementation(libs.commons.collections4) implementation(libs.commons.io) implementation(libs.commons.lang3) @@ -163,6 +179,7 @@ tasks.test { doFirst { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.12") + environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.2") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalog.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalog.java index 52f7e6cffe2..9ec01954f0d 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalog.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogBackend.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogBackend.java index 767c5c8ff87..d430cff8585 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogBackend.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogBackend.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogCapability.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogCapability.java index dbaa85b09d6..e076a1fd0ca 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogCapability.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogCapability.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java index 99003f5c892..402c864cbf0 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; @@ -89,7 +103,10 @@ public void initialize( Map resultConf = Maps.newHashMap(prefixMap); resultConf.putAll(gravitinoConfig); - + resultConf.put("catalog_uuid", info.id().toString()); + if (!resultConf.containsKey(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND_NAME)) { + resultConf.put(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND_NAME, info.name()); + } IcebergConfig icebergConfig = new IcebergConfig(resultConf); this.icebergTableOps = new IcebergTableOps(icebergConfig); diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java index eab79da21e1..2f44caa36b9 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java @@ -1,12 +1,28 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; import static com.datastrato.gravitino.connector.PropertyEntry.enumImmutablePropertyEntry; import static com.datastrato.gravitino.connector.PropertyEntry.stringRequiredPropertyEntry; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.AuthenticationConfig; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos.KerberosConfig; import com.datastrato.gravitino.connector.BaseCatalogPropertiesMetadata; import com.datastrato.gravitino.connector.PropertyEntry; import com.google.common.collect.ImmutableList; @@ -17,7 +33,7 @@ import java.util.Map; public class IcebergCatalogPropertiesMetadata extends BaseCatalogPropertiesMetadata { - public static final String CATALOG_BACKEND_NAME = "catalog-backend"; + public static final String CATALOG_BACKEND = "catalog-backend"; public static final String GRAVITINO_JDBC_USER = "jdbc-user"; public static final String ICEBERG_JDBC_USER = "jdbc.user"; @@ -29,17 +45,17 @@ public class IcebergCatalogPropertiesMetadata extends BaseCatalogPropertiesMetad public static final String GRAVITINO_JDBC_DRIVER = "jdbc-driver"; public static final String WAREHOUSE = "warehouse"; public static final String URI = "uri"; + public static final String CATALOG_BACKEND_NAME = "catalog-backend-name"; private static final Map> PROPERTIES_METADATA; // Map that maintains the mapping of keys in Gravitino to that in Iceberg, for example, users // will only need to set the configuration 'catalog-backend' in Gravitino and Gravitino will - // change - // it to `catalogType` automatically and pass it to Iceberg. + // change it to `catalogType` automatically and pass it to Iceberg. public static final Map GRAVITINO_CONFIG_TO_ICEBERG = ImmutableMap.of( - CATALOG_BACKEND_NAME, - CATALOG_BACKEND_NAME, + CATALOG_BACKEND, + CATALOG_BACKEND, GRAVITINO_JDBC_DRIVER, GRAVITINO_JDBC_DRIVER, GRAVITINO_JDBC_USER, @@ -49,13 +65,30 @@ public class IcebergCatalogPropertiesMetadata extends BaseCatalogPropertiesMetad URI, URI, WAREHOUSE, - WAREHOUSE); + WAREHOUSE, + CATALOG_BACKEND_NAME, + CATALOG_BACKEND_NAME); + + public static final Map KERBEROS_CONFIGURATION_FOR_HIVE_BACKEND = + ImmutableMap.of( + KerberosConfig.PRINCIPAL_KEY, + KerberosConfig.PRINCIPAL_KEY, + KerberosConfig.KET_TAB_URI_KEY, + KerberosConfig.KET_TAB_URI_KEY, + KerberosConfig.CHECK_INTERVAL_SEC_KEY, + KerberosConfig.CHECK_INTERVAL_SEC_KEY, + KerberosConfig.FETCH_TIMEOUT_SEC_KEY, + KerberosConfig.FETCH_TIMEOUT_SEC_KEY, + AuthenticationConfig.IMPERSONATION_ENABLE_KEY, + AuthenticationConfig.IMPERSONATION_ENABLE_KEY, + AuthenticationConfig.AUTH_TYPE_KEY, + AuthenticationConfig.AUTH_TYPE_KEY); static { List> propertyEntries = ImmutableList.of( enumImmutablePropertyEntry( - CATALOG_BACKEND_NAME, + CATALOG_BACKEND, "Iceberg catalog type choose properties", true, IcebergCatalogBackend.class, @@ -67,6 +100,8 @@ public class IcebergCatalogPropertiesMetadata extends BaseCatalogPropertiesMetad WAREHOUSE, "Iceberg catalog warehouse config", false, false)); HashMap> result = Maps.newHashMap(BASIC_CATALOG_PROPERTY_ENTRIES); result.putAll(Maps.uniqueIndex(propertyEntries, PropertyEntry::getName)); + result.putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES); + result.putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES); PROPERTIES_METADATA = ImmutableMap.copyOf(result); } @@ -82,6 +117,10 @@ public Map transformProperties(Map properties) { if (GRAVITINO_CONFIG_TO_ICEBERG.containsKey(key)) { gravitinoConfig.put(GRAVITINO_CONFIG_TO_ICEBERG.get(key), value); } + + if (KERBEROS_CONFIGURATION_FOR_HIVE_BACKEND.containsKey(key)) { + gravitinoConfig.put(KERBEROS_CONFIGURATION_FOR_HIVE_BACKEND.get(key), value); + } }); return gravitinoConfig; } diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergColumn.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergColumn.java index 90e08038940..c8218ada057 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergColumn.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergColumn.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java index f8d998703ff..eec8c579af0 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java @@ -1,11 +1,24 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; -import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.CATALOG_BACKEND_NAME; import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.GRAVITINO_JDBC_DRIVER; import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_INITIALIZE; import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_PASSWORD; @@ -22,12 +35,13 @@ import com.datastrato.gravitino.server.web.OverwriteDefaultConfig; import com.google.common.collect.ImmutableMap; import java.util.Map; +import java.util.Optional; import org.apache.commons.lang3.StringUtils; public class IcebergConfig extends Config implements OverwriteDefaultConfig { public static final ConfigEntry CATALOG_BACKEND = - new ConfigBuilder(CATALOG_BACKEND_NAME) + new ConfigBuilder(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND) .doc("Catalog backend of Gravitino Iceberg catalog") .version(ConfigConstants.VERSION_0_2_0) .stringConf() @@ -103,10 +117,21 @@ public class IcebergConfig extends Config implements OverwriteDefaultConfig { .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(1000); + public static final ConfigEntry CATALOG_BACKEND_NAME = + new ConfigBuilder(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND_NAME) + .doc("The catalog name for Iceberg catalog backend") + .version(ConfigConstants.VERSION_0_5_2) + .stringConf() + .create(); + public String getJdbcDriver() { return get(JDBC_DRIVER); } + public String getCatalogBackendName(String defaultCatalogBackendName) { + return Optional.ofNullable(get(CATALOG_BACKEND_NAME)).orElse(defaultCatalogBackendName); + } + public IcebergConfig(Map properties) { super(false); loadFromMap(properties, k -> true); diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergHiveCachedClientPool.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergHiveCachedClientPool.java new file mode 100644 index 00000000000..d779bf1b5fe --- /dev/null +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergHiveCachedClientPool.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package com.datastrato.gravitino.catalog.lakehouse.iceberg; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Scheduler; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import java.io.Closeable; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.ClientPool; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.hive.HiveClientPool; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.util.ThreadPools; +import org.apache.thrift.TException; + +/** + * Referred from Apache Iceberg's CachedClientPool implementation + * hive-metastore/src/main/java/org/apache/iceberg/hive/CachedClientPool.java + * + *

IcebergHiveCachedClientPool is used for every Iceberg catalog with Hive backend, I changed the + * method clientPool() from + * + * 

{@code
+ * HiveClientPool clientPool() {
+ *    return clientPoolCache.get(key, k -> new HiveClientPool(clientPoolSize, conf));
+ *  }
+ * }
+ * + * to + * + * 
{@code
+ * HiveClientPool clientPool() {
+ *   Key key = extractKey(properties.get(CatalogProperties.CLIENT_POOL_CACHE_KEYS), conf);
+ *   return clientPoolCache.get(key, k -> new HiveClientPool(clientPoolSize, conf));
+ * }
+ * }
+ * + * Why do we need to do this? Because the original client pool in iceberg uses a fixed username to + * create the client pool (please see the key in the method clientPool()). Assuming the original + * name is A and when a new user B tries to call the clientPool() method, it will use the connection + * that belongs to A. This will not work with kerberos authentication as it will change the user + * name. + */ +public class IcebergHiveCachedClientPool + implements ClientPool, Closeable { + private static final String CONF_ELEMENT_PREFIX = "conf:"; + + private static Cache clientPoolCache; + + private final Configuration conf; + private final Map properties; + private final int clientPoolSize; + private final long evictionInterval; + private ScheduledExecutorService scheduledExecutorService; + + public IcebergHiveCachedClientPool(Configuration conf, Map properties) { + this.conf = conf; + this.clientPoolSize = + PropertyUtil.propertyAsInt( + properties, + CatalogProperties.CLIENT_POOL_SIZE, + CatalogProperties.CLIENT_POOL_SIZE_DEFAULT); + this.evictionInterval = + PropertyUtil.propertyAsLong( + properties, + CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS, + CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS_DEFAULT); + this.properties = properties; + init(); + } + + @VisibleForTesting + HiveClientPool clientPool() { + Key key = extractKey(properties.get(CatalogProperties.CLIENT_POOL_CACHE_KEYS), conf); + return clientPoolCache.get(key, k -> new HiveClientPool(clientPoolSize, conf)); + } + + private synchronized void init() { + if (clientPoolCache == null) { + // Since Caffeine does not ensure that removalListener will be involved after expiration + // We use a scheduler with one thread to clean up expired clients. + scheduledExecutorService = ThreadPools.newScheduledPool("hive-metastore-cleaner", 1); + clientPoolCache = + Caffeine.newBuilder() + .expireAfterAccess(evictionInterval, TimeUnit.MILLISECONDS) + .removalListener((ignored, value, cause) -> ((HiveClientPool) value).close()) + .scheduler(Scheduler.forScheduledExecutorService(scheduledExecutorService)) + .build(); + } + } + + @VisibleForTesting + static Cache clientPoolCache() { + return clientPoolCache; + } + + @Override + public R run(Action action) + throws TException, InterruptedException { + return clientPool().run(action); + } + + @Override + public R run(Action action, boolean retry) + throws TException, InterruptedException { + return clientPool().run(action, retry); + } + + @VisibleForTesting + static Key extractKey(String cacheKeys, Configuration conf) { + // generate key elements in a certain order, so that the Key instances are comparable + List elements = Lists.newArrayList(); + elements.add(conf.get(HiveConf.ConfVars.METASTOREURIS.varname, "")); + elements.add(conf.get("HIVE_CONF_CATALOG", "hive")); + if (cacheKeys == null || cacheKeys.isEmpty()) { + return Key.of(elements); + } + + Set types = Sets.newTreeSet(Comparator.comparingInt(Enum::ordinal)); + Map confElements = Maps.newTreeMap(); + for (String element : cacheKeys.split(",", -1)) { + String trimmed = element.trim(); + if (trimmed.toLowerCase(Locale.ROOT).startsWith(CONF_ELEMENT_PREFIX)) { + String key = trimmed.substring(CONF_ELEMENT_PREFIX.length()); + ValidationException.check( + !confElements.containsKey(key), "Conf key element %s already specified", key); + confElements.put(key, conf.get(key)); + } else { + KeyElementType type = KeyElementType.valueOf(trimmed.toUpperCase()); + switch (type) { + case UGI: + case USER_NAME: + ValidationException.check( + !types.contains(type), "%s key element already specified", type.name()); + types.add(type); + break; + default: + throw new ValidationException("Unknown key element %s", trimmed); + } + } + } + for (KeyElementType type : types) { + switch (type) { + case UGI: + try { + elements.add(UserGroupInformation.getCurrentUser()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + break; + case USER_NAME: + try { + elements.add(UserGroupInformation.getCurrentUser().getUserName()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + break; + default: + throw new RuntimeException("Unexpected key element " + type.name()); + } + } + return Key.of(elements); + } + + static class Key { + private final List elements; + + List elements() { + return elements; + } + + public Key(List elements) { + this.elements = elements; + } + + static Key of(List elements) { + return new Key(elements); + } + } + + private enum KeyElementType { + UGI, + USER_NAME, + CONF + } + + @Override + public void close() throws IOException { + clientPoolCache.asMap().forEach((key, value) -> value.close()); + clientPoolCache.invalidateAll(); + scheduledExecutorService.shutdownNow(); + } +} diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java index 7a3517f6152..99d4167bd74 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchema.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchema.java index 37e2487e22e..54570b73a3e 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchema.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchema.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchemaPropertiesMetadata.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchemaPropertiesMetadata.java index 86d4b21c662..3b3ded361df 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchemaPropertiesMetadata.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchemaPropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTable.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTable.java index 58740b8860b..874e23a0c51 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTable.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTable.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java index 9dbe9d3ea02..70aa5583771 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/AuthenticationConfig.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/AuthenticationConfig.java new file mode 100644 index 00000000000..4fe2d5d3bf8 --- /dev/null +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/AuthenticationConfig.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication; + +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos.KerberosConfig.DEFAULT_IMPERSONATION_ENABLE; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.config.ConfigBuilder; +import com.datastrato.gravitino.config.ConfigConstants; +import com.datastrato.gravitino.config.ConfigEntry; +import com.datastrato.gravitino.connector.PropertyEntry; +import com.google.common.collect.ImmutableMap; +import java.util.Map; + +public class AuthenticationConfig extends Config { + + // The key for the authentication type, currently we support Kerberos and simple + public static final String AUTH_TYPE_KEY = "authentication.type"; + + public static final String IMPERSONATION_ENABLE_KEY = "authentication.impersonation-enable"; + + public static final boolean DEFAULT_IMPERSONATION_ENABLE = false; + + enum AuthenticationType { + SIMPLE, + KERBEROS + } + + public AuthenticationConfig(Map properties) { + super(false); + loadFromMap(properties, k -> true); + } + + public static final ConfigEntry AUTH_TYPE_ENTRY = + new ConfigBuilder(AUTH_TYPE_KEY) + .doc( + "The type of authentication for Iceberg catalog, currently we support simple and Kerberos") + .version(ConfigConstants.VERSION_0_5_1) + .stringConf() + .createWithDefault("simple"); + + public static final ConfigEntry ENABLE_IMPERSONATION_ENTRY = + new ConfigBuilder(IMPERSONATION_ENABLE_KEY) + .doc("Whether to enable impersonation for Iceberg catalog") + .version(ConfigConstants.VERSION_0_5_1) + .booleanConf() + .createWithDefault(DEFAULT_IMPERSONATION_ENABLE); + + public String getAuthType() { + return get(AUTH_TYPE_ENTRY); + } + + public boolean isSimpleAuth() { + return AuthenticationType.SIMPLE.name().equalsIgnoreCase(getAuthType()); + } + + public boolean isKerberosAuth() { + return AuthenticationType.KERBEROS.name().equalsIgnoreCase(getAuthType()); + } + + public boolean isImpersonationEnabled() { + return get(ENABLE_IMPERSONATION_ENTRY); + } + + public static final Map> AUTHENTICATION_PROPERTY_ENTRIES = + new ImmutableMap.Builder>() + .put( + IMPERSONATION_ENABLE_KEY, + PropertyEntry.booleanPropertyEntry( + IMPERSONATION_ENABLE_KEY, + "Whether to enable impersonation for the Iceberg catalog", + false, + true, + DEFAULT_IMPERSONATION_ENABLE, + false, + false)) + .put( + AUTH_TYPE_KEY, + PropertyEntry.stringImmutablePropertyEntry( + AUTH_TYPE_KEY, + "The type of authentication for Hadoop catalog, currently we support simple Kerberos", + false, + "simple", + false, + false)) + .build(); +} diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/kerberos/FetchFileUtils.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/kerberos/FetchFileUtils.java new file mode 100644 index 00000000000..a6f470de812 --- /dev/null +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/kerberos/FetchFileUtils.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.util.Optional; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +public class FetchFileUtils { + + private FetchFileUtils() {} + + public static void fetchFileFromUri( + String fileUri, File destFile, int timeout, Configuration conf) throws IOException { + try { + URI uri = new URI(fileUri); + String scheme = Optional.ofNullable(uri.getScheme()).orElse("file"); + + switch (scheme) { + case "http": + case "https": + case "ftp": + FileUtils.copyURLToFile(uri.toURL(), destFile, timeout * 1000, timeout * 1000); + break; + + case "file": + Files.createSymbolicLink(destFile.toPath(), new File(uri.getPath()).toPath()); + break; + + case "hdfs": + FileSystem.get(conf).copyToLocalFile(new Path(uri), new Path(destFile.toURI())); + break; + + default: + throw new IllegalArgumentException( + String.format("Doesn't support the scheme %s", scheme)); + } + } catch (URISyntaxException ue) { + throw new IllegalArgumentException("The uri of file has the wrong format", ue); + } + } +} diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/kerberos/HiveBackendProxy.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/kerberos/HiveBackendProxy.java new file mode 100644 index 00000000000..08f02f3b33f --- /dev/null +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/kerberos/HiveBackendProxy.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos; + +import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergHiveCachedClientPool; +import com.datastrato.gravitino.utils.PrincipalUtils; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.security.PrivilegedExceptionAction; +import java.util.Map; +import net.sf.cglib.proxy.Enhancer; +import net.sf.cglib.proxy.MethodInterceptor; +import net.sf.cglib.proxy.MethodProxy; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.iceberg.ClientPool; +import org.apache.iceberg.hive.HiveCatalog; +import org.apache.thrift.TException; + +/** + * Proxy class for HiveCatalog to support kerberos authentication. We can also make HiveCatalog as a + * generic type and pass it as a parameter to the constructor. + */ +public class HiveBackendProxy implements MethodInterceptor { + + private final HiveCatalog target; + private final String kerberosRealm; + private final UserGroupInformation proxyUser; + private final Map properties; + private final ClientPool newClientPool; + + public HiveBackendProxy( + Map properties, HiveCatalog target, String kerberosRealm) { + this.target = target; + this.properties = properties; + this.kerberosRealm = kerberosRealm; + try { + proxyUser = UserGroupInformation.getCurrentUser(); + + // Replace the original client pool with IcebergHiveCachedClientPool. Why do we need to do + // this? Because the original client pool in iceberg uses a fixed username to create the + // client pool, and it will not work with kerberos authentication. We need to create a new + // client pool with the current user. For more, please see CachedClientPool#clientPool and + // notice the value of `key` + this.newClientPool = resetIcebergHiveClientPool(); + } catch (IOException e) { + throw new RuntimeException("Failed to get current user", e); + } catch (IllegalAccessException | NoSuchFieldException e) { + throw new RuntimeException("Failed to reset IcebergHiveClientPool", e); + } + } + + @Override + public Object intercept(Object o, Method method, Object[] objects, MethodProxy methodProxy) + throws Throwable { + + String proxyKerberosPrincipalName = PrincipalUtils.getCurrentPrincipal().getName(); + if (!proxyKerberosPrincipalName.contains("@")) { + proxyKerberosPrincipalName = + String.format("%s@%s", proxyKerberosPrincipalName, kerberosRealm); + } + + UserGroupInformation realUser = + UserGroupInformation.createProxyUser(proxyKerberosPrincipalName, proxyUser); + + String token = + newClientPool.run( + client -> + client.getDelegationToken( + PrincipalUtils.getCurrentPrincipal().getName(), proxyUser.getShortUserName())); + + Token delegationToken = new Token<>(); + delegationToken.decodeFromUrlString(token); + realUser.addToken(delegationToken); + + return realUser.doAs( + (PrivilegedExceptionAction) + () -> { + try { + return methodProxy.invoke(target, objects); + } catch (Throwable e) { + if (RuntimeException.class.isAssignableFrom(e.getClass())) { + throw (RuntimeException) e; + } + throw new RuntimeException("Failed to invoke method", e); + } + }); + } + + private ClientPool resetIcebergHiveClientPool() + throws IllegalAccessException, NoSuchFieldException { + final Field m = HiveCatalog.class.getDeclaredField("clients"); + m.setAccessible(true); + + // TODO: we need to close the original client pool and thread pool, or it will cause memory + // leak. + ClientPool newClientPool = + new IcebergHiveCachedClientPool(target.getConf(), properties); + m.set(target, newClientPool); + return newClientPool; + } + + public HiveCatalog getProxy() { + Enhancer e = new Enhancer(); + e.setClassLoader(target.getClass().getClassLoader()); + e.setSuperclass(target.getClass()); + e.setCallback(this); + return (HiveCatalog) e.create(); + } +} diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/kerberos/KerberosClient.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/kerberos/KerberosClient.java new file mode 100644 index 00000000000..3110d13d54b --- /dev/null +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/kerberos/KerberosClient.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos; + +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KerberosClient { + private static final Logger LOG = LoggerFactory.getLogger(KerberosClient.class); + + private final ScheduledThreadPoolExecutor checkTgtExecutor; + private final Map conf; + private final Configuration hadoopConf; + + public KerberosClient(Map conf, Configuration hadoopConf) { + this.conf = conf; + this.hadoopConf = hadoopConf; + this.checkTgtExecutor = new ScheduledThreadPoolExecutor(1, getThreadFactory("check-tgt")); + } + + public String login(String keytabFilePath) throws IOException { + KerberosConfig kerberosConfig = new KerberosConfig(conf); + + // Check the principal and keytab file + String catalogPrincipal = kerberosConfig.getPrincipalName(); + Preconditions.checkArgument( + StringUtils.isNotBlank(catalogPrincipal), "The principal can't be blank"); + @SuppressWarnings("null") + List principalComponents = Splitter.on('@').splitToList(catalogPrincipal); + Preconditions.checkArgument( + principalComponents.size() == 2, "The principal has the wrong format"); + + // Login + UserGroupInformation.setConfiguration(hadoopConf); + KerberosName.resetDefaultRealm(); + UserGroupInformation.loginUserFromKeytab(catalogPrincipal, keytabFilePath); + UserGroupInformation kerberosLoginUgi = UserGroupInformation.getCurrentUser(); + + // Refresh the cache if it's out of date. + int checkInterval = kerberosConfig.getCheckIntervalSec(); + checkTgtExecutor.scheduleAtFixedRate( + () -> { + try { + kerberosLoginUgi.checkTGTAndReloginFromKeytab(); + } catch (Exception e) { + LOG.error("Fail to refresh ugi token: ", e); + } + }, + checkInterval, + checkInterval, + TimeUnit.SECONDS); + + return principalComponents.get(1); + } + + public File saveKeyTabFileFromUri(Long catalogId) throws IOException { + + KerberosConfig kerberosConfig = new KerberosConfig(conf); + + String keyTabUri = kerberosConfig.getKeytab(); + Preconditions.checkArgument(StringUtils.isNotBlank(keyTabUri), "Keytab uri can't be blank"); + // TODO: Support to download the file from Kerberos HDFS + Preconditions.checkArgument( + !keyTabUri.trim().startsWith("hdfs"), "Keytab uri doesn't support to use HDFS"); + + File keytabsDir = new File("keytabs"); + if (!keytabsDir.exists()) { + // Ignore the return value, because there exists many Hive catalog operations making + // this directory. + keytabsDir.mkdir(); + } + + File keytabFile = new File(String.format(KerberosConfig.GRAVITINO_KEYTAB_FORMAT, catalogId)); + keytabFile.deleteOnExit(); + if (keytabFile.exists() && !keytabFile.delete()) { + throw new IllegalStateException( + String.format("Fail to delete keytab file %s", keytabFile.getAbsolutePath())); + } + + // TODO: Make the configuration + int fetchKeytabFileTimeout = kerberosConfig.getFetchTimeoutSec(); + FetchFileUtils.fetchFileFromUri(keyTabUri, keytabFile, fetchKeytabFileTimeout, hadoopConf); + + return keytabFile; + } + + private static ThreadFactory getThreadFactory(String factoryName) { + return new ThreadFactoryBuilder().setDaemon(true).setNameFormat(factoryName + "-%d").build(); + } +} diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/kerberos/KerberosConfig.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/kerberos/KerberosConfig.java new file mode 100644 index 00000000000..e63b047fcff --- /dev/null +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/authentication/kerberos/KerberosConfig.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos; + +import com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.AuthenticationConfig; +import com.datastrato.gravitino.config.ConfigBuilder; +import com.datastrato.gravitino.config.ConfigConstants; +import com.datastrato.gravitino.config.ConfigEntry; +import com.datastrato.gravitino.connector.PropertyEntry; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; + +public class KerberosConfig extends AuthenticationConfig { + public static final String KET_TAB_URI_KEY = "authentication.kerberos.keytab-uri"; + + public static final String PRINCIPAL_KEY = "authentication.kerberos.principal"; + + public static final String CHECK_INTERVAL_SEC_KEY = "authentication.kerberos.check-interval-sec"; + + public static final String FETCH_TIMEOUT_SEC_KEY = + "authentication.kerberos.keytab-fetch-timeout-sec"; + + public static final String GRAVITINO_KEYTAB_FORMAT = "keytabs/gravitino-%s-keytab"; + + public static final ConfigEntry PRINCIPAL_ENTRY = + new ConfigBuilder(PRINCIPAL_KEY) + .doc("The principal of the Kerberos for Iceberg catalog with Kerberos authentication") + .version(ConfigConstants.VERSION_0_6_0) + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + public static final ConfigEntry KEYTAB_ENTRY = + new ConfigBuilder(KET_TAB_URI_KEY) + .doc("The keytab of the Kerberos for Iceberg catalog with Kerberos authentication") + .version(ConfigConstants.VERSION_0_6_0) + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + public static final ConfigEntry CHECK_INTERVAL_SEC_ENTRY = + new ConfigBuilder(CHECK_INTERVAL_SEC_KEY) + .doc( + "The check interval of the Kerberos credential for Iceberg catalog with Kerberos authentication") + .version(ConfigConstants.VERSION_0_6_0) + .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) + .createWithDefault(2); + + public static final ConfigEntry FETCH_TIMEOUT_SEC_ENTRY = + new ConfigBuilder(FETCH_TIMEOUT_SEC_KEY) + .doc( + "The fetch timeout of the Kerberos key table of Iceberg catalog with Kerberos authentication") + .version(ConfigConstants.VERSION_0_6_0) + .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) + .createWithDefault(2); + + public KerberosConfig(Map properties) { + super(properties); + loadFromMap(properties, k -> true); + } + + public boolean isImpersonationEnabled() { + return get(ENABLE_IMPERSONATION_ENTRY); + } + + public String getPrincipalName() { + return get(PRINCIPAL_ENTRY); + } + + public String getKeytab() { + return get(KEYTAB_ENTRY); + } + + public int getCheckIntervalSec() { + return get(CHECK_INTERVAL_SEC_ENTRY); + } + + public int getFetchTimeoutSec() { + return get(FETCH_TIMEOUT_SEC_ENTRY); + } + + public static final Map> KERBEROS_PROPERTY_ENTRIES = + new ImmutableMap.Builder>() + .put( + KET_TAB_URI_KEY, + PropertyEntry.stringImmutablePropertyEntry( + KET_TAB_URI_KEY, + "The keytab of the Kerberos for Iceberg catalog with Kerberos authentication", + false, + null, + false, + false)) + .put( + PRINCIPAL_KEY, + PropertyEntry.stringImmutablePropertyEntry( + PRINCIPAL_KEY, + "The principal of the Kerberos for Iceberg catalog with Kerberos authentication", + false, + null, + false, + false)) + .put( + CHECK_INTERVAL_SEC_KEY, + PropertyEntry.integerOptionalPropertyEntry( + CHECK_INTERVAL_SEC_KEY, + "The check interval of the Kerberos credential for Iceberg catalog with Kerberos authentication", + true, + 60, + false)) + .put( + FETCH_TIMEOUT_SEC_KEY, + PropertyEntry.integerOptionalPropertyEntry( + FETCH_TIMEOUT_SEC_KEY, + "The fetch timeout of the Kerberos key table of Iceberg catalog with Kerberos authentication", + true, + 60, + false)) + .build(); +} diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ConvertUtil.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ConvertUtil.java index c50c9d7aa38..99b94c7418f 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ConvertUtil.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ConvertUtil.java @@ -1,15 +1,30 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.converter.IcebergDataTypeConverter.CONVERTER; + import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergColumn; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTable; import java.util.Arrays; import org.apache.iceberg.Schema; import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; public class ConvertUtil { @@ -28,26 +43,6 @@ public static Schema toIcebergSchema(IcebergTable gravitinoTable) { return new Schema(converted.asNestedType().asStructType().fields()); } - /** - * Convert the Gravitino type to the Iceberg type. - * - * @param gravitinoType Gravitino type. - * @return Iceberg type. - */ - public static Type toIcebergType(com.datastrato.gravitino.rel.types.Type gravitinoType) { - return ToIcebergTypeVisitor.visit(gravitinoType, new ToIcebergType()); - } - - /** - * Convert the nested type of Iceberg to the type of gravitino. - * - * @param type Iceberg type of field. - * @return Gravitino type. - */ - public static com.datastrato.gravitino.rel.types.Type formIcebergType(Type type) { - return TypeUtil.visit(type, new FromIcebergType()); - } - /** * Convert the nested field of Iceberg to the Iceberg column. * @@ -59,7 +54,7 @@ public static IcebergColumn fromNestedField(Types.NestedField nestedField) { .withName(nestedField.name()) .withNullable(nestedField.isOptional()) .withComment(nestedField.doc()) - .withType(ConvertUtil.formIcebergType(nestedField.type())) + .withType(CONVERTER.toGravitino(nestedField.type())) .build(); } diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergPartitionSpec.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergPartitionSpec.java index d1346599215..e27f6867bb8 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergPartitionSpec.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergPartitionSpec.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergSortOrder.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergSortOrder.java index 758c4e9a409..d602d2c11d8 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergSortOrder.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergSortOrder.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergType.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergType.java index 0caadc22fad..39fd103482b 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergType.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/FromIcebergType.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/IcebergDataTypeConverter.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/IcebergDataTypeConverter.java new file mode 100644 index 00000000000..005f1fc4d33 --- /dev/null +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/IcebergDataTypeConverter.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; + +import com.datastrato.gravitino.connector.DataTypeConverter; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; + +public class IcebergDataTypeConverter implements DataTypeConverter { + public static final IcebergDataTypeConverter CONVERTER = new IcebergDataTypeConverter(); + + @Override + public Type fromGravitino(com.datastrato.gravitino.rel.types.Type gravitinoType) { + return ToIcebergTypeVisitor.visit(gravitinoType, new ToIcebergType()); + } + + @Override + public com.datastrato.gravitino.rel.types.Type toGravitino(Type type) { + return TypeUtil.visit(type, new FromIcebergType()); + } +} diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergPartitionSpec.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergPartitionSpec.java index 5c5cde03dc7..1bc3439a0e9 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergPartitionSpec.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergPartitionSpec.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergSortOrder.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergSortOrder.java index d58bac39d91..16bc1f3e1b4 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergSortOrder.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergSortOrder.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergType.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergType.java index f4f37bcfccc..e435f2f963c 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergType.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergType.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergTypeVisitor.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergTypeVisitor.java index d356d17d069..06b93208c9e 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergTypeVisitor.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/ToIcebergTypeVisitor.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java index 8288d615600..d3bb37137d2 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.ops; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOpsHelper.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOpsHelper.java index 6c87ee98a49..945926924d8 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOpsHelper.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOpsHelper.java @@ -1,12 +1,27 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.ops; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.converter.IcebergDataTypeConverter.CONVERTER; + import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.catalog.lakehouse.iceberg.converter.ConvertUtil; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.TableChange.AddColumn; import com.datastrato.gravitino.rel.TableChange.After; @@ -150,8 +165,7 @@ private void doUpdateColumnType( fieldName); icebergTableSchema.findField(fieldName).isOptional(); - org.apache.iceberg.types.Type type = - ConvertUtil.toIcebergType(updateColumnType.getNewDataType()); + org.apache.iceberg.types.Type type = CONVERTER.fromGravitino(updateColumnType.getNewDataType()); Preconditions.checkArgument( type.isPrimitiveType(), "Cannot update %s, not a primitive type: %s", fieldName, type); icebergUpdateSchema.updateColumn(fieldName, (PrimitiveType) type); @@ -199,7 +213,7 @@ private void doAddColumn( icebergUpdateSchema.addColumn( getParentName(addColumn.fieldName()), getLeafName(addColumn.fieldName()), - ConvertUtil.toIcebergType(addColumn.getDataType()), + CONVERTER.fromGravitino(addColumn.getDataType()), addColumn.getComment()); } else { // TODO: figure out how to enable users to add required columns @@ -207,7 +221,7 @@ private void doAddColumn( icebergUpdateSchema.addRequiredColumn( getParentName(addColumn.fieldName()), getLeafName(addColumn.fieldName()), - ConvertUtil.toIcebergType(addColumn.getDataType()), + CONVERTER.fromGravitino(addColumn.getDataType()), addColumn.getComment()); } diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java index f55c6be0ba7..d577d533e0b 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java @@ -1,16 +1,38 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.utils; import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_INITIALIZE; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.AuthenticationConfig; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos.HiveBackendProxy; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos.KerberosClient; +import java.io.File; +import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.Catalog; @@ -26,10 +48,12 @@ public class IcebergCatalogUtil { private static final Logger LOG = LoggerFactory.getLogger(IcebergCatalogUtil.class); private static InMemoryCatalog loadMemoryCatalog(Map properties) { + IcebergConfig icebergConfig = new IcebergConfig(properties); + String icebergCatalogName = icebergConfig.getCatalogBackendName("memory"); InMemoryCatalog memoryCatalog = new InMemoryCatalog(); Map resultProperties = new HashMap<>(properties); resultProperties.put(CatalogProperties.WAREHOUSE_LOCATION, "/tmp"); - memoryCatalog.initialize("memory", resultProperties); + memoryCatalog.initialize(icebergCatalogName, resultProperties); return memoryCatalog; } @@ -37,14 +61,52 @@ private static HiveCatalog loadHiveCatalog(Map properties) { HiveCatalog hiveCatalog = new HiveCatalog(); HdfsConfiguration hdfsConfiguration = new HdfsConfiguration(); properties.forEach(hdfsConfiguration::set); - hiveCatalog.setConf(hdfsConfiguration); - hiveCatalog.initialize("hive", properties); - return hiveCatalog; + IcebergConfig icebergConfig = new IcebergConfig(properties); + String icebergCatalogName = icebergConfig.getCatalogBackendName("hive"); + + AuthenticationConfig authenticationConfig = new AuthenticationConfig(properties); + if (authenticationConfig.isSimpleAuth()) { + hiveCatalog.setConf(hdfsConfiguration); + hiveCatalog.initialize(icebergCatalogName, properties); + return hiveCatalog; + } else if (authenticationConfig.isKerberosAuth()) { + Map resultProperties = new HashMap<>(properties); + resultProperties.put(CatalogProperties.CLIENT_POOL_CACHE_KEYS, "USER_NAME"); + hdfsConfiguration.set(HADOOP_SECURITY_AUTHORIZATION, "true"); + hdfsConfiguration.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + hiveCatalog.setConf(hdfsConfiguration); + hiveCatalog.initialize(icebergCatalogName, properties); + + String realm = initKerberosAndReturnRealm(properties, hdfsConfiguration); + if (authenticationConfig.isImpersonationEnabled()) { + HiveBackendProxy proxyHiveCatalog = + new HiveBackendProxy(resultProperties, hiveCatalog, realm); + return proxyHiveCatalog.getProxy(); + } + + return hiveCatalog; + } else { + throw new UnsupportedOperationException( + "Unsupported authentication method: " + authenticationConfig.getAuthType()); + } + } + + private static String initKerberosAndReturnRealm( + Map properties, Configuration conf) { + try { + KerberosClient kerberosClient = new KerberosClient(properties, conf); + File keytabFile = + kerberosClient.saveKeyTabFileFromUri(Long.valueOf(properties.get("catalog_uuid"))); + return kerberosClient.login(keytabFile.getAbsolutePath()); + } catch (IOException e) { + throw new RuntimeException("Failed to login with kerberos", e); + } } private static JdbcCatalog loadJdbcCatalog(Map properties) { IcebergConfig icebergConfig = new IcebergConfig(properties); String driverClassName = icebergConfig.getJdbcDriver(); + String icebergCatalogName = icebergConfig.getCatalogBackendName("jdbc"); icebergConfig.get(IcebergConfig.JDBC_USER); icebergConfig.get(IcebergConfig.JDBC_PASSWORD); @@ -63,16 +125,18 @@ private static JdbcCatalog loadJdbcCatalog(Map properties) { HdfsConfiguration hdfsConfiguration = new HdfsConfiguration(); properties.forEach(hdfsConfiguration::set); jdbcCatalog.setConf(hdfsConfiguration); - jdbcCatalog.initialize("jdbc", properties); + jdbcCatalog.initialize(icebergCatalogName, properties); return jdbcCatalog; } private static Catalog loadRestCatalog(Map properties) { + IcebergConfig icebergConfig = new IcebergConfig(properties); + String icebergCatalogName = icebergConfig.getCatalogBackendName("rest"); RESTCatalog restCatalog = new RESTCatalog(); HdfsConfiguration hdfsConfiguration = new HdfsConfiguration(); properties.forEach(hdfsConfiguration::set); restCatalog.setConf(hdfsConfiguration); - restCatalog.initialize("rest", properties); + restCatalog.initialize(icebergCatalogName, properties); return restCatalog; } diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergObjectMapper.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergObjectMapper.java index d98b655d6c8..6a6c67909f7 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergObjectMapper.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergObjectMapper.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergObjectMapperProvider.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergObjectMapperProvider.java index b74afa025ff..3f2666e4e64 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergObjectMapperProvider.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergObjectMapperProvider.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergRestUtils.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergRestUtils.java index 5e3f33dc661..3db4853052b 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergRestUtils.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergRestUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/DummyMetricsStore.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/DummyMetricsStore.java index f1dcb0b1538..6a1bb01ea18 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/DummyMetricsStore.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/DummyMetricsStore.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.metrics; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/IcebergMetricsFormatter.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/IcebergMetricsFormatter.java index 18ffcdf5526..5baad33d45e 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/IcebergMetricsFormatter.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/IcebergMetricsFormatter.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.metrics; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/IcebergMetricsManager.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/IcebergMetricsManager.java index 1f3f993a369..4f01b7a2fc3 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/IcebergMetricsManager.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/IcebergMetricsManager.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.metrics; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/IcebergMetricsStore.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/IcebergMetricsStore.java index 16d2400e8fd..425a50b6015 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/IcebergMetricsStore.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/IcebergMetricsStore.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.metrics; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergConfigOperations.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergConfigOperations.java index 56d0ad5bc6c..7929d67d181 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergConfigOperations.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergConfigOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.rest; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergNamespaceOperations.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergNamespaceOperations.java index 135f8e07572..a357c801501 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergNamespaceOperations.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergNamespaceOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.rest; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergTableOperations.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergTableOperations.java index 169941d3f7a..f2539e8be32 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergTableOperations.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergTableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.rest; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergTableRenameOperations.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergTableRenameOperations.java index 0f6ef3c4af4..aff210b2275 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergTableRenameOperations.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergTableRenameOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.rest; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider b/catalogs/catalog-lakehouse-iceberg/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider index 25be79ab2c3..0ef9cc6f719 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider +++ b/catalogs/catalog-lakehouse-iceberg/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider @@ -1,5 +1,19 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalog \ No newline at end of file diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService b/catalogs/catalog-lakehouse-iceberg/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService index fa3ff960adb..a5dbec4de49 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService +++ b/catalogs/catalog-lakehouse-iceberg/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService @@ -1,5 +1,19 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergRESTService diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/resources/core-site.xml.template b/catalogs/catalog-lakehouse-iceberg/src/main/resources/core-site.xml.template index 263112f905f..efa7db5adb6 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/resources/core-site.xml.template +++ b/catalogs/catalog-lakehouse-iceberg/src/main/resources/core-site.xml.template @@ -1,6 +1,20 @@ diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/resources/hdfs-site.xml.template b/catalogs/catalog-lakehouse-iceberg/src/main/resources/hdfs-site.xml.template index 4b7c0376bae..ced4a08b486 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/resources/hdfs-site.xml.template +++ b/catalogs/catalog-lakehouse-iceberg/src/main/resources/hdfs-site.xml.template @@ -1,6 +1,20 @@ \ No newline at end of file diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/resources/lakehouse-iceberg.conf b/catalogs/catalog-lakehouse-iceberg/src/main/resources/lakehouse-iceberg.conf index 82b425ee341..acc9304f29d 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/resources/lakehouse-iceberg.conf +++ b/catalogs/catalog-lakehouse-iceberg/src/main/resources/lakehouse-iceberg.conf @@ -1,6 +1,20 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # ## This file holds common configurations for Lakehouse-iceberg catalog. The format of the key is diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java index 11b158fc30f..d41d8ea0cce 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; @@ -100,7 +114,7 @@ void testCatalogProperty() { try (IcebergCatalogOperations ops = new IcebergCatalogOperations()) { ops.initialize(conf, entity.toCatalogInfo(), ICEBERG_PROPERTIES_METADATA); Map map1 = Maps.newHashMap(); - map1.put(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND_NAME, "test"); + map1.put(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND, "test"); PropertiesMetadata metadata = ICEBERG_PROPERTIES_METADATA.catalogPropertiesMetadata(); Assertions.assertThrows( IllegalArgumentException.class, @@ -109,7 +123,7 @@ void testCatalogProperty() { }); Map map2 = Maps.newHashMap(); - map2.put(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND_NAME, "hive"); + map2.put(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND, "hive"); map2.put(IcebergCatalogPropertiesMetadata.URI, "127.0.0.1"); map2.put(IcebergCatalogPropertiesMetadata.WAREHOUSE, "test"); Assertions.assertDoesNotThrow( @@ -124,7 +138,7 @@ void testCatalogProperty() { () -> PropertiesMetadataHelpers.validatePropertyForCreate(metadata, map3)); Assertions.assertTrue( - throwable.getMessage().contains(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND_NAME)); + throwable.getMessage().contains(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND)); } } } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergConfig.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergConfig.java index d6b26003220..086e747044e 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergConfig.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergConfig.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergSchema.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergSchema.java index ad28a9d6c2f..58a01f8f9b5 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergSchema.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergSchema.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergTable.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergTable.java index 5a87c10f585..7d4a85026c2 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergTable.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergTable.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java index 58e72cde9de..8f2b7452d44 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestConvertUtil.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestConvertUtil.java index b5784543dd7..c127656b4ba 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestConvertUtil.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestConvertUtil.java @@ -1,9 +1,25 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.converter.IcebergDataTypeConverter.CONVERTER; + import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergColumn; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTable; import com.datastrato.gravitino.meta.AuditInfo; @@ -89,7 +105,7 @@ public void testToPrimitiveType() { ByteType byteType = ByteType.get(); IllegalArgumentException exception = Assertions.assertThrows( - IllegalArgumentException.class, () -> ConvertUtil.toIcebergType(byteType)); + IllegalArgumentException.class, () -> CONVERTER.fromGravitino(byteType)); Assertions.assertTrue( exception .getMessage() @@ -98,63 +114,63 @@ public void testToPrimitiveType() { ShortType shortType = ShortType.get(); exception = Assertions.assertThrows( - IllegalArgumentException.class, () -> ConvertUtil.toIcebergType(shortType)); + IllegalArgumentException.class, () -> CONVERTER.fromGravitino(shortType)); Assertions.assertTrue( exception .getMessage() .contains("Iceberg do not support Byte and Short Type, use Integer instead")); Assertions.assertTrue( - ConvertUtil.toIcebergType(com.datastrato.gravitino.rel.types.Types.BooleanType.get()) + CONVERTER.fromGravitino(com.datastrato.gravitino.rel.types.Types.BooleanType.get()) instanceof Types.BooleanType); Assertions.assertTrue( - ConvertUtil.toIcebergType(com.datastrato.gravitino.rel.types.Types.StringType.get()) + CONVERTER.fromGravitino(com.datastrato.gravitino.rel.types.Types.StringType.get()) instanceof Types.StringType); Assertions.assertTrue( - ConvertUtil.toIcebergType(com.datastrato.gravitino.rel.types.Types.IntegerType.get()) + CONVERTER.fromGravitino(com.datastrato.gravitino.rel.types.Types.IntegerType.get()) instanceof Types.IntegerType); Assertions.assertTrue( - ConvertUtil.toIcebergType(com.datastrato.gravitino.rel.types.Types.LongType.get()) + CONVERTER.fromGravitino(com.datastrato.gravitino.rel.types.Types.LongType.get()) instanceof Types.LongType); Assertions.assertTrue( - ConvertUtil.toIcebergType(com.datastrato.gravitino.rel.types.Types.FloatType.get()) + CONVERTER.fromGravitino(com.datastrato.gravitino.rel.types.Types.FloatType.get()) instanceof Types.FloatType); Assertions.assertTrue( - ConvertUtil.toIcebergType(com.datastrato.gravitino.rel.types.Types.DoubleType.get()) + CONVERTER.fromGravitino(com.datastrato.gravitino.rel.types.Types.DoubleType.get()) instanceof Types.DoubleType); Assertions.assertTrue( - ConvertUtil.toIcebergType(com.datastrato.gravitino.rel.types.Types.DateType.get()) + CONVERTER.fromGravitino(com.datastrato.gravitino.rel.types.Types.DateType.get()) instanceof Types.DateType); Assertions.assertTrue( - ConvertUtil.toIcebergType(com.datastrato.gravitino.rel.types.Types.TimeType.get()) + CONVERTER.fromGravitino(com.datastrato.gravitino.rel.types.Types.TimeType.get()) instanceof Types.TimeType); Assertions.assertTrue( - ConvertUtil.toIcebergType(com.datastrato.gravitino.rel.types.Types.BinaryType.get()) + CONVERTER.fromGravitino(com.datastrato.gravitino.rel.types.Types.BinaryType.get()) instanceof Types.BinaryType); Assertions.assertTrue( - ConvertUtil.toIcebergType(com.datastrato.gravitino.rel.types.Types.UUIDType.get()) + CONVERTER.fromGravitino(com.datastrato.gravitino.rel.types.Types.UUIDType.get()) instanceof Types.UUIDType); Type timestampTZ = - ConvertUtil.toIcebergType( + CONVERTER.fromGravitino( com.datastrato.gravitino.rel.types.Types.TimestampType.withTimeZone()); Assertions.assertTrue(timestampTZ instanceof Types.TimestampType); Assertions.assertTrue(((Types.TimestampType) timestampTZ).shouldAdjustToUTC()); Type timestamp = - ConvertUtil.toIcebergType( + CONVERTER.fromGravitino( com.datastrato.gravitino.rel.types.Types.TimestampType.withoutTimeZone()); Assertions.assertTrue(timestamp instanceof Types.TimestampType); Assertions.assertFalse(((Types.TimestampType) timestamp).shouldAdjustToUTC()); Type decimalType = - ConvertUtil.toIcebergType(com.datastrato.gravitino.rel.types.Types.DecimalType.of(9, 2)); + CONVERTER.fromGravitino(com.datastrato.gravitino.rel.types.Types.DecimalType.of(9, 2)); Assertions.assertTrue(decimalType instanceof Types.DecimalType); Assertions.assertEquals(9, ((Types.DecimalType) decimalType).precision()); Assertions.assertEquals(2, ((Types.DecimalType) decimalType).scale()); Type fixedCharType = - ConvertUtil.toIcebergType(com.datastrato.gravitino.rel.types.Types.FixedType.of(9)); + CONVERTER.fromGravitino(com.datastrato.gravitino.rel.types.Types.FixedType.of(9)); Assertions.assertTrue(fixedCharType instanceof Types.FixedType); Assertions.assertEquals(9, ((Types.FixedType) fixedCharType).length()); @@ -163,14 +179,14 @@ public void testToPrimitiveType() { com.datastrato.gravitino.rel.types.Types.StringType.get(), com.datastrato.gravitino.rel.types.Types.IntegerType.get(), true); - Type convertedMapType = ConvertUtil.toIcebergType(mapType); + Type convertedMapType = CONVERTER.fromGravitino(mapType); Assertions.assertTrue(convertedMapType instanceof Types.MapType); Assertions.assertTrue(((Types.MapType) convertedMapType).keyType() instanceof Types.StringType); Assertions.assertTrue( ((Types.MapType) convertedMapType).valueType() instanceof Types.IntegerType); Type listType = - ConvertUtil.toIcebergType( + CONVERTER.fromGravitino( com.datastrato.gravitino.rel.types.Types.ListType.of( com.datastrato.gravitino.rel.types.Types.FloatType.get(), true)); Assertions.assertTrue(listType instanceof Types.ListType); @@ -180,7 +196,7 @@ public void testToPrimitiveType() { @Test public void testToNestedType() { Type listTypeNullable = - ConvertUtil.toIcebergType( + CONVERTER.fromGravitino( com.datastrato.gravitino.rel.types.Types.ListType.of( com.datastrato.gravitino.rel.types.Types.FloatType.get(), true)); Assertions.assertTrue(listTypeNullable instanceof Types.ListType); @@ -188,7 +204,7 @@ public void testToNestedType() { Assertions.assertTrue(listTypeNullable.asListType().isElementOptional()); Type listTypeNotNull = - ConvertUtil.toIcebergType( + CONVERTER.fromGravitino( com.datastrato.gravitino.rel.types.Types.ListType.of( com.datastrato.gravitino.rel.types.Types.FloatType.get(), false)); Assertions.assertTrue(listTypeNotNull instanceof Types.ListType); @@ -196,7 +212,7 @@ public void testToNestedType() { Assertions.assertTrue(listTypeNotNull.asListType().isElementRequired()); Type mapTypeNullable = - ConvertUtil.toIcebergType( + CONVERTER.fromGravitino( com.datastrato.gravitino.rel.types.Types.MapType.of( com.datastrato.gravitino.rel.types.Types.StringType.get(), com.datastrato.gravitino.rel.types.Types.IntegerType.get(), @@ -207,7 +223,7 @@ public void testToNestedType() { Assertions.assertTrue(mapTypeNullable.asMapType().isValueOptional()); Type mapTypeNotNull = - ConvertUtil.toIcebergType( + CONVERTER.fromGravitino( com.datastrato.gravitino.rel.types.Types.MapType.of( com.datastrato.gravitino.rel.types.Types.StringType.get(), com.datastrato.gravitino.rel.types.Types.IntegerType.get(), @@ -218,7 +234,7 @@ public void testToNestedType() { Assertions.assertTrue(mapTypeNotNull.asMapType().isValueRequired()); Type structTypeNullable = - ConvertUtil.toIcebergType( + CONVERTER.fromGravitino( com.datastrato.gravitino.rel.types.Types.StructType.of( com.datastrato.gravitino.rel.types.Types.StructType.Field.nullableField( "col1", @@ -243,7 +259,7 @@ public void testToNestedType() { structTypeNullable.asStructType().fields().get(1).type().asListType().isElementOptional()); Type structTypeNotNull = - ConvertUtil.toIcebergType( + CONVERTER.fromGravitino( com.datastrato.gravitino.rel.types.Types.StructType.of( com.datastrato.gravitino.rel.types.Types.StructType.Field.notNullField( "col1", @@ -270,49 +286,49 @@ public void testToNestedType() { @Test public void testFormIcebergType() { Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.StringType.get()) + CONVERTER.toGravitino(Types.StringType.get()) instanceof com.datastrato.gravitino.rel.types.Types.StringType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.BinaryType.get()) + CONVERTER.toGravitino(Types.BinaryType.get()) instanceof com.datastrato.gravitino.rel.types.Types.BinaryType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.TimeType.get()) + CONVERTER.toGravitino(Types.TimeType.get()) instanceof com.datastrato.gravitino.rel.types.Types.TimeType); com.datastrato.gravitino.rel.types.Type TimestampTypeWithoutZone = - ConvertUtil.formIcebergType(Types.TimestampType.withoutZone()); + CONVERTER.toGravitino(Types.TimestampType.withoutZone()); Assertions.assertTrue( TimestampTypeWithoutZone instanceof com.datastrato.gravitino.rel.types.Types.TimestampType); Assertions.assertFalse( ((com.datastrato.gravitino.rel.types.Types.TimestampType) TimestampTypeWithoutZone) .hasTimeZone()); com.datastrato.gravitino.rel.types.Type TimestampTypeWithZone = - ConvertUtil.formIcebergType(Types.TimestampType.withZone()); + CONVERTER.toGravitino(Types.TimestampType.withZone()); Assertions.assertTrue( TimestampTypeWithZone instanceof com.datastrato.gravitino.rel.types.Types.TimestampType); Assertions.assertTrue( ((com.datastrato.gravitino.rel.types.Types.TimestampType) TimestampTypeWithZone) .hasTimeZone()); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.DoubleType.get()) + CONVERTER.toGravitino(Types.DoubleType.get()) instanceof com.datastrato.gravitino.rel.types.Types.DoubleType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.FloatType.get()) + CONVERTER.toGravitino(Types.FloatType.get()) instanceof com.datastrato.gravitino.rel.types.Types.FloatType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.IntegerType.get()) + CONVERTER.toGravitino(Types.IntegerType.get()) instanceof com.datastrato.gravitino.rel.types.Types.IntegerType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.LongType.get()) + CONVERTER.toGravitino(Types.LongType.get()) instanceof com.datastrato.gravitino.rel.types.Types.LongType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.UUIDType.get()) + CONVERTER.toGravitino(Types.UUIDType.get()) instanceof com.datastrato.gravitino.rel.types.Types.UUIDType); Assertions.assertTrue( - ConvertUtil.formIcebergType(Types.DateType.get()) + CONVERTER.toGravitino(Types.DateType.get()) instanceof com.datastrato.gravitino.rel.types.Types.DateType); com.datastrato.gravitino.rel.types.Type decimalType = - ConvertUtil.formIcebergType(Types.DecimalType.of(9, 2)); + CONVERTER.toGravitino(Types.DecimalType.of(9, 2)); Assertions.assertTrue( decimalType instanceof com.datastrato.gravitino.rel.types.Types.DecimalType); Assertions.assertEquals( @@ -321,14 +337,14 @@ public void testFormIcebergType() { 2, ((com.datastrato.gravitino.rel.types.Types.DecimalType) decimalType).scale()); com.datastrato.gravitino.rel.types.Type fixedType = - ConvertUtil.formIcebergType(Types.FixedType.ofLength(2)); + CONVERTER.toGravitino(Types.FixedType.ofLength(2)); Assertions.assertTrue(fixedType instanceof com.datastrato.gravitino.rel.types.Types.FixedType); Assertions.assertEquals( 2, ((com.datastrato.gravitino.rel.types.Types.FixedType) fixedType).length()); Types.MapType mapType = Types.MapType.ofOptional(1, 2, Types.StringType.get(), Types.IntegerType.get()); - com.datastrato.gravitino.rel.types.Type gravitinoMapType = ConvertUtil.formIcebergType(mapType); + com.datastrato.gravitino.rel.types.Type gravitinoMapType = CONVERTER.toGravitino(mapType); Assertions.assertTrue( gravitinoMapType instanceof com.datastrato.gravitino.rel.types.Types.MapType); Assertions.assertTrue( @@ -339,8 +355,7 @@ public void testFormIcebergType() { instanceof com.datastrato.gravitino.rel.types.Types.IntegerType); Types.ListType listType = Types.ListType.ofOptional(1, Types.StringType.get()); - com.datastrato.gravitino.rel.types.Type gravitinoListType = - ConvertUtil.formIcebergType(listType); + com.datastrato.gravitino.rel.types.Type gravitinoListType = CONVERTER.toGravitino(listType); Assertions.assertTrue( gravitinoListType instanceof com.datastrato.gravitino.rel.types.Types.ListType); Assertions.assertTrue( @@ -357,8 +372,7 @@ public void testFormIcebergType() { Types.StructType.of( Types.NestedField.optional(0, "integer_type", Types.IntegerType.get(), "integer type"), Types.NestedField.optional(1, "struct_type", structTypeInside, "struct type inside")); - com.datastrato.gravitino.rel.types.Type gravitinoStructType = - ConvertUtil.formIcebergType(structType); + com.datastrato.gravitino.rel.types.Type gravitinoStructType = CONVERTER.toGravitino(structType); // check for type Assertions.assertTrue( (gravitinoStructType) instanceof com.datastrato.gravitino.rel.types.Types.StructType); @@ -497,7 +511,7 @@ public void testFromNestedField() { } private static void checkType(Type type, com.datastrato.gravitino.rel.types.Type expected) { - com.datastrato.gravitino.rel.types.Type actual = ConvertUtil.formIcebergType(type); + com.datastrato.gravitino.rel.types.Type actual = CONVERTER.toGravitino(type); checkType(actual, expected); } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestFromIcebergPartitionSpec.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestFromIcebergPartitionSpec.java index a4d2c2f6e59..07cba523249 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestFromIcebergPartitionSpec.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestFromIcebergPartitionSpec.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestFromIcebergSortOrder.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestFromIcebergSortOrder.java index c852277bdd5..e134b24fa96 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestFromIcebergSortOrder.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestFromIcebergSortOrder.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestToIcebergPartitionSpec.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestToIcebergPartitionSpec.java index eaf11d6701d..e4afb67f7c2 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestToIcebergPartitionSpec.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestToIcebergPartitionSpec.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestToIcebergSortOrder.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestToIcebergSortOrder.java index 32cd6920da7..478de1f88bd 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestToIcebergSortOrder.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/TestToIcebergSortOrder.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.converter; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java index a2d2fd3ea60..bc306d565e4 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.integration.test; @@ -120,11 +134,16 @@ public void startup() throws Exception { @AfterAll public void stop() throws Exception { - clearTableAndSchema(); - metalake.dropCatalog(catalogName); - client.dropMetalake(metalakeName); - spark.close(); - AbstractIT.stopIntegrationTest(); + try { + clearTableAndSchema(); + metalake.dropCatalog(catalogName); + client.dropMetalake(metalakeName); + } finally { + if (spark != null) { + spark.close(); + } + AbstractIT.stopIntegrationTest(); + } } @AfterEach @@ -163,7 +182,7 @@ private void initSparkEnv() { private void clearTableAndSchema() { if (catalog.asSchemas().schemaExists(schemaName)) { NameIdentifier[] nameIdentifiers = - catalog.asTableCatalog().listTables(Namespace.of(metalakeName, catalogName, schemaName)); + catalog.asTableCatalog().listTables(Namespace.of(schemaName)); for (NameIdentifier nameIdentifier : nameIdentifiers) { catalog.asTableCatalog().dropTable(nameIdentifier); } @@ -187,14 +206,18 @@ private void createCatalog() { Map catalogProperties = Maps.newHashMap(); catalogProperties.put("key1", "val1"); catalogProperties.put("key2", "val2"); + String icebergCatalogBackendName = "iceberg-catalog-name-test"; catalogProperties.put(IcebergConfig.CATALOG_BACKEND.getKey(), TYPE); catalogProperties.put(IcebergConfig.CATALOG_URI.getKey(), URIS); catalogProperties.put(IcebergConfig.CATALOG_WAREHOUSE.getKey(), WAREHOUSE); + catalogProperties.put(IcebergConfig.CATALOG_BACKEND_NAME.getKey(), icebergCatalogBackendName); Map icebergCatalogProperties = Maps.newHashMap(); icebergCatalogProperties.put(IcebergConfig.CATALOG_URI.getKey(), URIS); icebergCatalogProperties.put(IcebergConfig.CATALOG_WAREHOUSE.getKey(), WAREHOUSE); + icebergCatalogProperties.put( + IcebergConfig.CATALOG_BACKEND_NAME.getKey(), icebergCatalogBackendName); icebergCatalog = IcebergCatalogUtil.loadCatalogBackend(TYPE, icebergCatalogProperties); if (icebergCatalog instanceof SupportsNamespaces) { @@ -211,7 +234,7 @@ private void createCatalog() { } private void createSchema() { - NameIdentifier ident = NameIdentifier.of(metalakeName, catalogName, schemaName); + NameIdentifier ident = NameIdentifier.of(schemaName); Map prop = Maps.newHashMap(); prop.put("key1", "val1"); prop.put("key2", "val2"); @@ -263,7 +286,7 @@ void testOperationIcebergSchema() { // create schema check. String testSchemaName = GravitinoITUtils.genRandomName("test_schema_1"); - NameIdentifier schemaIdent = NameIdentifier.of(metalakeName, catalogName, testSchemaName); + NameIdentifier schemaIdent = NameIdentifier.of(testSchemaName); schemas.createSchema(schemaIdent.name(), schema_comment, Collections.emptyMap()); schemaNames = new HashSet<>(Arrays.asList(schemas.listSchemas())); @@ -309,8 +332,7 @@ void testOperationIcebergSchema() { TableCatalog tableCatalog = catalog.asTableCatalog(); // create failed check. - NameIdentifier table = - NameIdentifier.of(metalakeName, catalogName, testSchemaName, "test_table"); + NameIdentifier table = NameIdentifier.of(testSchemaName, "test_table"); Assertions.assertThrows( NoSuchSchemaException.class, () -> @@ -341,8 +363,7 @@ void testOperationIcebergSchema() { @Test void testCreateTableWithNullComment() { Column[] columns = createColumns(); - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); TableCatalog tableCatalog = catalog.asTableCatalog(); Table createdTable = @@ -358,8 +379,7 @@ void testCreateAndLoadIcebergTable() { // Create table from Gravitino API Column[] columns = createColumns(); - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Distribution distribution = Distributions.NONE; final SortOrder[] sortOrders = @@ -461,8 +481,7 @@ void testTimestampTypeConversion() { String timestampTableName = "timestamp_table"; - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, timestampTableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, timestampTableName); Map properties = createProperties(); TableCatalog tableCatalog = catalog.asTableCatalog(); @@ -509,7 +528,7 @@ void testTimestampTypeConversion() { void testListAndDropIcebergTable() { Column[] columns = createColumns(); - NameIdentifier table1 = NameIdentifier.of(metalakeName, catalogName, schemaName, "table_1"); + NameIdentifier table1 = NameIdentifier.of(schemaName, "table_1"); Map properties = createProperties(); TableCatalog tableCatalog = catalog.asTableCatalog(); @@ -521,8 +540,7 @@ void testListAndDropIcebergTable() { Transforms.EMPTY_TRANSFORM, Distributions.NONE, new SortOrder[0]); - NameIdentifier[] nameIdentifiers = - tableCatalog.listTables(Namespace.of(metalakeName, catalogName, schemaName)); + NameIdentifier[] nameIdentifiers = tableCatalog.listTables(Namespace.of(schemaName)); Assertions.assertEquals(1, nameIdentifiers.length); Assertions.assertEquals("table_1", nameIdentifiers[0].name()); @@ -531,7 +549,7 @@ void testListAndDropIcebergTable() { Assertions.assertEquals(1, tableIdentifiers.size()); Assertions.assertEquals("table_1", tableIdentifiers.get(0).name()); - NameIdentifier table2 = NameIdentifier.of(metalakeName, catalogName, schemaName, "table_2"); + NameIdentifier table2 = NameIdentifier.of(schemaName, "table_2"); tableCatalog.createTable( table2, columns, @@ -540,7 +558,7 @@ void testListAndDropIcebergTable() { Transforms.EMPTY_TRANSFORM, Distributions.NONE, new SortOrder[0]); - nameIdentifiers = tableCatalog.listTables(Namespace.of(metalakeName, catalogName, schemaName)); + nameIdentifiers = tableCatalog.listTables(Namespace.of(schemaName)); Assertions.assertEquals(2, nameIdentifiers.length); Assertions.assertEquals("table_1", nameIdentifiers[0].name()); Assertions.assertEquals("table_2", nameIdentifiers[1].name()); @@ -553,13 +571,12 @@ void testListAndDropIcebergTable() { Assertions.assertDoesNotThrow(() -> tableCatalog.dropTable(table1)); - nameIdentifiers = tableCatalog.listTables(Namespace.of(metalakeName, catalogName, schemaName)); + nameIdentifiers = tableCatalog.listTables(Namespace.of(schemaName)); Assertions.assertEquals(1, nameIdentifiers.length); Assertions.assertEquals("table_2", nameIdentifiers[0].name()); Assertions.assertDoesNotThrow(() -> tableCatalog.dropTable(table2)); - Namespace schemaNamespace = Namespace.of(metalakeName, catalogName, schemaName); - nameIdentifiers = tableCatalog.listTables(schemaNamespace); + nameIdentifiers = tableCatalog.listTables(Namespace.of(schemaName)); Assertions.assertEquals(0, nameIdentifiers.length); tableIdentifiers = @@ -574,7 +591,7 @@ public void testAlterIcebergTable() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), columns, table_comment, createProperties(), @@ -587,7 +604,7 @@ public void testAlterIcebergTable() { catalog .asTableCatalog() .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), TableChange.rename(alertTableName), TableChange.updateComment(table_comment + "_new")); }); @@ -596,15 +613,14 @@ public void testAlterIcebergTable() { catalog .asTableCatalog() .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), - TableChange.rename(alertTableName)); + NameIdentifier.of(schemaName, tableName), TableChange.rename(alertTableName)); Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().lastModifier()); Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().creator()); catalog .asTableCatalog() .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, alertTableName), + NameIdentifier.of(schemaName, alertTableName), TableChange.updateComment(table_comment + "_new"), TableChange.removeProperty("key1"), TableChange.setProperty("key2", "val2_new"), @@ -614,10 +630,7 @@ public void testAlterIcebergTable() { TableChange.updateColumnType( new String[] {ICEBERG_COL_NAME1}, Types.IntegerType.get())); - table = - catalog - .asTableCatalog() - .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, alertTableName)); + table = catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName, alertTableName)); Assertions.assertEquals(alertTableName, table.name()); Assertions.assertEquals("val2_new", table.properties().get("key2")); @@ -656,9 +669,7 @@ public void testAlterIcebergTable() { () -> catalog .asTableCatalog() - .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, alertTableName), - withDefaultValue)); + .alterTable(NameIdentifier.of(schemaName, alertTableName), withDefaultValue)); Assertions.assertTrue( exception .getMessage() @@ -671,11 +682,7 @@ public void testAlterIcebergTable() { Column[] newColumns = new Column[] {col1, col2, col3}; NameIdentifier tableIdentifier = - NameIdentifier.of( - metalakeName, - catalogName, - schemaName, - GravitinoITUtils.genRandomName("CatalogHiveIT_table")); + NameIdentifier.of(schemaName, GravitinoITUtils.genRandomName("CatalogHiveIT_table")); catalog .asTableCatalog() .createTable( @@ -754,7 +761,7 @@ void testPartitionAndSortOrderIcebergTable() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, testTableName), + NameIdentifier.of(schemaName, testTableName), columns, table_comment, createProperties(), @@ -803,7 +810,7 @@ void testOperationDataIcebergTable() { catalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, testTableName), + NameIdentifier.of(schemaName, testTableName), columns, table_comment, createProperties(), @@ -889,7 +896,7 @@ void testOperationDataIcebergTable() { @Test public void testOperatorSchemeProperties() { - NameIdentifier ident = NameIdentifier.of(metalakeName, catalogName, "testCreateSchemaCheck"); + NameIdentifier ident = NameIdentifier.of("testCreateSchemaCheck"); Map prop = Maps.newHashMap(); prop.put(IcebergSchemaPropertiesMetadata.COMMENT, "val1"); prop.put("key2", "val2"); @@ -935,8 +942,7 @@ public void testOperatorSchemeProperties() { public void testTableDistribution() { Column[] columns = createColumns(); - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Distribution distribution = Distributions.NONE; final SortOrder[] sortOrders = @@ -1080,8 +1086,7 @@ void testIcebergTablePropertiesWhenCreate() { // Create table from Gravitino API Column[] columns = createColumns(); - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Distribution distribution = Distributions.NONE; final SortOrder[] sortOrders = @@ -1168,8 +1173,7 @@ private static void checkIcebergTableFileFormat( public void testTableSortOrder() { Column[] columns = createColumns(); - NameIdentifier tableIdentifier = - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); Distribution distribution = Distributions.NONE; final SortOrder[] sortOrders = diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergHiveIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergHiveIT.java index bd335ab61bc..a9a4b501da6 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergHiveIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergHiveIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.integration.test; @@ -8,7 +22,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestInstance; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class CatalogIcebergHiveIT extends CatalogIcebergBaseIT { diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java new file mode 100644 index 00000000000..3adb6668968 --- /dev/null +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.iceberg.integration.test; + +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.AuthenticationConfig.AUTH_TYPE_KEY; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos.KerberosConfig.IMPERSONATION_ENABLE_KEY; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos.KerberosConfig.KET_TAB_URI_KEY; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos.KerberosConfig.PRINCIPAL_KEY; +import static com.datastrato.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; + +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; +import com.datastrato.gravitino.client.GravitinoAdminClient; +import com.datastrato.gravitino.client.GravitinoMetalake; +import com.datastrato.gravitino.client.KerberosTokenProvider; +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.container.HiveContainer; +import com.datastrato.gravitino.integration.test.util.AbstractIT; +import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrders; +import com.datastrato.gravitino.rel.expressions.transforms.Transforms; +import com.datastrato.gravitino.rel.types.Types; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.hadoop.security.authentication.util.KerberosUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Tag("gravitino-docker-test") +public class CatalogIcebergKerberosHiveIT extends AbstractIT { + + private static final Logger LOG = LoggerFactory.getLogger(CatalogIcebergKerberosHiveIT.class); + + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + + private static final String SDK_KERBEROS_PRINCIPAL_KEY = "client.kerberos.principal"; + private static final String SDK_KERBEROS_KEYTAB_KEY = "client.kerberos.keytab"; + + private static final String GRAVITINO_CLIENT_PRINCIPAL = "gravitino_client@HADOOPKRB"; + private static final String GRAVITINO_CLIENT_KEYTAB = "/gravitino_client.keytab"; + + private static final String GRAVITINO_SERVER_PRINCIPAL = "HTTP/localhost@HADOOPKRB"; + private static final String GRAVITINO_SERVER_KEYTAB = "/gravitino_server.keytab"; + + private static final String HIVE_METASTORE_CLIENT_PRINCIPAL = "cli@HADOOPKRB"; + private static final String HIVE_METASTORE_CLIENT_KEYTAB = "/client.keytab"; + + private static String TMP_DIR; + + private static HiveContainer kerberosHiveContainer; + + private static GravitinoAdminClient adminClient; + + private static final String METALAKE_NAME = GravitinoITUtils.genRandomName("test_metalake"); + private static final String CATALOG_NAME = GravitinoITUtils.genRandomName("test_catalog"); + private static final String SCHEMA_NAME = GravitinoITUtils.genRandomName("test_schema"); + private static final String TABLE_NAME = GravitinoITUtils.genRandomName("test_table"); + + private static String URIS; + private static String TYPE; + private static String WAREHOUSE; + + private static final String HIVE_COL_NAME1 = "col1"; + private static final String HIVE_COL_NAME2 = "col2"; + private static final String HIVE_COL_NAME3 = "col3"; + + @BeforeAll + public static void startIntegrationTest() { + containerSuite.startKerberosHiveContainer(); + kerberosHiveContainer = containerSuite.getKerberosHiveContainer(); + + URIS = + String.format( + "thrift://%s:%d", + kerberosHiveContainer.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); + TYPE = "hive"; + WAREHOUSE = + String.format( + "hdfs://%s:%d/user/hive/warehouse-catalog-iceberg/", + kerberosHiveContainer.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); + + try { + File baseDir = new File(System.getProperty("java.io.tmpdir")); + File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); + file.deleteOnExit(); + TMP_DIR = file.getAbsolutePath(); + + // Prepare kerberos related-config; + prepareKerberosConfig(); + + // Config kerberos configuration for Gravitino server + addKerberosConfig(); + + ignoreIcebergRestService = false; + // Start Gravitino server + AbstractIT.startIntegrationTest(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @AfterAll + public static void stop() { + // Reset the UGI + UserGroupInformation.reset(); + + LOG.info("krb5 path: {}", System.getProperty("java.security.krb5.conf")); + // Clean up the kerberos configuration + System.clearProperty("java.security.krb5.conf"); + System.clearProperty("sun.security.krb5.debug"); + + AbstractIT.client = null; + } + + private static void prepareKerberosConfig() throws Exception { + // Keytab of the Gravitino SDK client + kerberosHiveContainer + .getContainer() + .copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB); + + // Keytab of the Gravitino server + kerberosHiveContainer + .getContainer() + .copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB); + + // Keytab of Gravitino server to connector to Hive + kerberosHiveContainer + .getContainer() + .copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB); + + String tmpKrb5Path = TMP_DIR + "/krb5.conf_tmp"; + String krb5Path = TMP_DIR + "/krb5.conf"; + kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); + + // Modify the krb5.conf and change the kdc and admin_server to the container IP + String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress(); + String content = FileUtils.readFileToString(new File(tmpKrb5Path), StandardCharsets.UTF_8); + content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88"); + content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749"); + FileUtils.write(new File(krb5Path), content, StandardCharsets.UTF_8); + + LOG.info("Kerberos kdc config:\n{}, path: {}", content, krb5Path); + System.setProperty("java.security.krb5.conf", krb5Path); + System.setProperty("sun.security.krb5.debug", "true"); + + refreshKerberosConfig(); + KerberosName.resetDefaultRealm(); + + LOG.info("Kerberos default realm: {}", KerberosUtil.getDefaultRealm()); + } + + private static void refreshKerberosConfig() { + Class classRef; + try { + if (System.getProperty("java.vendor").contains("IBM")) { + classRef = Class.forName("com.ibm.security.krb5.internal.Config"); + } else { + classRef = Class.forName("sun.security.krb5.Config"); + } + + Method refershMethod = classRef.getMethod("refresh"); + refershMethod.invoke(null); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static void addKerberosConfig() { + AbstractIT.customConfigs.put("gravitino.authenticator", "kerberos"); + AbstractIT.customConfigs.put( + "gravitino.authenticator.kerberos.principal", GRAVITINO_SERVER_PRINCIPAL); + AbstractIT.customConfigs.put( + "gravitino.authenticator.kerberos.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB); + AbstractIT.customConfigs.put(SDK_KERBEROS_KEYTAB_KEY, TMP_DIR + GRAVITINO_CLIENT_KEYTAB); + AbstractIT.customConfigs.put(SDK_KERBEROS_PRINCIPAL_KEY, GRAVITINO_CLIENT_PRINCIPAL); + } + + @Test + void testIcebergWithKerberosAndUserImpersonation() throws IOException { + KerberosTokenProvider provider = + KerberosTokenProvider.builder() + .withClientPrincipal(GRAVITINO_CLIENT_PRINCIPAL) + .withKeyTabFile(new File(TMP_DIR + GRAVITINO_CLIENT_KEYTAB)) + .build(); + adminClient = GravitinoAdminClient.builder(serverUri).withKerberosAuth(provider).build(); + + GravitinoMetalake gravitinoMetalake = + adminClient.createMetalake(METALAKE_NAME, null, ImmutableMap.of()); + + // Create a catalog + Map properties = Maps.newHashMap(); + properties.put(IMPERSONATION_ENABLE_KEY, "true"); + properties.put(AUTH_TYPE_KEY, "kerberos"); + + properties.put(KET_TAB_URI_KEY, TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB); + properties.put(PRINCIPAL_KEY, HIVE_METASTORE_CLIENT_PRINCIPAL); + properties.put( + CATALOG_BYPASS_PREFIX + "hive.metastore.kerberos.principal", + "hive/_HOST@HADOOPKRB" + .replace("_HOST", containerSuite.getKerberosHiveContainer().getHostName())); + properties.put(CATALOG_BYPASS_PREFIX + "hive.metastore.sasl.enabled", "true"); + + properties.put(IcebergConfig.CATALOG_BACKEND.getKey(), TYPE); + properties.put(IcebergConfig.CATALOG_URI.getKey(), URIS); + properties.put(IcebergConfig.CATALOG_WAREHOUSE.getKey(), WAREHOUSE); + properties.put("location", "hdfs://localhost:9000/user/hive/warehouse-catalog-iceberg"); + + Catalog catalog = + gravitinoMetalake.createCatalog( + CATALOG_NAME, Catalog.Type.RELATIONAL, "lakehouse-iceberg", "comment", properties); + + // Test create schema + Exception exception = + Assertions.assertThrows( + Exception.class, + () -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", ImmutableMap.of())); + String exceptionMessage = Throwables.getStackTraceAsString(exception); + + // Make sure the real user is 'gravitino_client' + Assertions.assertTrue( + exceptionMessage.contains("Permission denied: user=gravitino_client, access=WRITE")); + + // Now try to permit the user to create the schema again + kerberosHiveContainer.executeInContainer( + "hadoop", "fs", "-mkdir", "/user/hive/warehouse-catalog-iceberg"); + kerberosHiveContainer.executeInContainer( + "hadoop", "fs", "-chmod", "-R", "777", "/user/hive/warehouse-catalog-iceberg"); + Assertions.assertDoesNotThrow( + () -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", ImmutableMap.of())); + + // Create table + NameIdentifier tableNameIdentifier = NameIdentifier.of(SCHEMA_NAME, TABLE_NAME); + catalog + .asTableCatalog() + .createTable( + tableNameIdentifier, + createColumns(), + "", + ImmutableMap.of(), + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + SortOrders.NONE); + + // Now try to alter the table + catalog.asTableCatalog().alterTable(tableNameIdentifier, TableChange.rename("new_table")); + NameIdentifier newTableIdentifier = NameIdentifier.of(SCHEMA_NAME, "new_table"); + + // Old table name should not exist + Assertions.assertFalse(catalog.asTableCatalog().tableExists(tableNameIdentifier)); + Assertions.assertTrue(catalog.asTableCatalog().tableExists(newTableIdentifier)); + + // Drop table + catalog.asTableCatalog().dropTable(newTableIdentifier); + Assertions.assertFalse(catalog.asTableCatalog().tableExists(newTableIdentifier)); + + // Drop schema + catalog.asSchemas().dropSchema(SCHEMA_NAME, false); + Assertions.assertFalse(catalog.asSchemas().schemaExists(SCHEMA_NAME)); + + // Drop catalog + Assertions.assertTrue(gravitinoMetalake.dropCatalog(CATALOG_NAME)); + } + + @Test + void testIcebergWithKerberos() { + KerberosTokenProvider provider = + KerberosTokenProvider.builder() + .withClientPrincipal(GRAVITINO_CLIENT_PRINCIPAL) + .withKeyTabFile(new File(TMP_DIR + GRAVITINO_CLIENT_KEYTAB)) + .build(); + adminClient = GravitinoAdminClient.builder(serverUri).withKerberosAuth(provider).build(); + + String metalakeName = GravitinoITUtils.genRandomName("test_metalake"); + GravitinoMetalake gravitinoMetalake = + adminClient.createMetalake(metalakeName, null, ImmutableMap.of()); + + // Create a catalog + Map properties = Maps.newHashMap(); + properties.put(AUTH_TYPE_KEY, "kerberos"); + // Not user impersonation here + + properties.put(KET_TAB_URI_KEY, TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB); + properties.put(PRINCIPAL_KEY, HIVE_METASTORE_CLIENT_PRINCIPAL); + properties.put( + CATALOG_BYPASS_PREFIX + "hive.metastore.kerberos.principal", + "hive/_HOST@HADOOPKRB" + .replace("_HOST", containerSuite.getKerberosHiveContainer().getHostName())); + properties.put(CATALOG_BYPASS_PREFIX + "hive.metastore.sasl.enabled", "true"); + + properties.put(IcebergConfig.CATALOG_BACKEND.getKey(), TYPE); + properties.put(IcebergConfig.CATALOG_URI.getKey(), URIS); + properties.put(IcebergConfig.CATALOG_WAREHOUSE.getKey(), WAREHOUSE); + properties.put("location", "hdfs://localhost:9000/user/hive/warehouse-catalog-iceberg"); + + Catalog catalog = + gravitinoMetalake.createCatalog( + CATALOG_NAME, Catalog.Type.RELATIONAL, "lakehouse-iceberg", "comment", properties); + + // Test create schema + Exception exception = + Assertions.assertThrows( + Exception.class, + () -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", ImmutableMap.of())); + String exceptionMessage = Throwables.getStackTraceAsString(exception); + + // Make sure the real user is 'cli' because no impersonation here. + Assertions.assertTrue(exceptionMessage.contains("Permission denied: user=cli, access=WRITE")); + } + + private static Column[] createColumns() { + Column col1 = Column.of(HIVE_COL_NAME1, Types.IntegerType.get(), "col_1_comment"); + Column col2 = Column.of(HIVE_COL_NAME2, Types.DateType.get(), "col_2_comment"); + Column col3 = Column.of(HIVE_COL_NAME3, Types.StringType.get(), "col_3_comment"); + return new Column[] {col1, col2, col3}; + } +} diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergRestIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergRestIT.java index 58365e0ed6a..b18899f48e7 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergRestIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergRestIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.integration.test; @@ -14,7 +28,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestInstance; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class CatalogIcebergRestIT extends CatalogIcebergBaseIT { diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTHiveCatalogIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTHiveCatalogIT.java index e37b8370ca4..bdd30032a3e 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTHiveCatalogIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTHiveCatalogIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.integration.test; @@ -17,10 +31,10 @@ import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; -// Hive&Jdbc catalog must be tested with gravitino-docker-it env, +// Hive&Jdbc catalog must be tested with gravitino-docker-test env, // so we should create a separate class instead using junit `parameterized test` // to auto-generate catalog type -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) public class IcebergRESTHiveCatalogIT extends IcebergRESTServiceIT { private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java index 3f4518f2510..5477e01c8af 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.integration.test; @@ -18,7 +32,7 @@ import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) public class IcebergRESTJdbcCatalogIT extends IcebergRESTServiceIT { private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTServiceBaseIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTServiceBaseIT.java index 5cdb6ddd9a5..a8f3ebe6618 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTServiceBaseIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTServiceBaseIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.integration.test; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTServiceIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTServiceIT.java index 3f717a426a4..2103853cccd 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTServiceIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/IcebergRESTServiceIT.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.integration.test; @@ -27,11 +41,13 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.api.condition.EnabledIf; +@Tag("gravitino-docker-test") @SuppressWarnings("FormatStringAnnotation") @TestInstance(Lifecycle.PER_CLASS) public class IcebergRESTServiceIT extends IcebergRESTServiceBaseIT { diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java index 9061a40315b..9eeace48f65 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.integration.test; @@ -30,7 +44,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class TestMultipleJDBCLoad extends AbstractIT { private static final TestDatabaseName TEST_DB_NAME = TestDatabaseName.PG_TEST_ICEBERG_CATALOG_MULTIPLE_JDBC_LOAD; @@ -109,7 +123,7 @@ public void testCreateMultipleJdbcInIceberg() throws URISyntaxException, SQLExce mysqlCatalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, mysqlCatalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), new Column[] {col1}, comment, Collections.emptyMap()); @@ -117,19 +131,14 @@ public void testCreateMultipleJdbcInIceberg() throws URISyntaxException, SQLExce postgreSqlCatalog .asTableCatalog() .createTable( - NameIdentifier.of(metalakeName, postgreSqlCatalogName, schemaName, tableName), + NameIdentifier.of(schemaName, tableName), new Column[] {col1}, comment, Collections.emptyMap()); Assertions.assertTrue( - mysqlCatalog - .asTableCatalog() - .tableExists(NameIdentifier.of(metalakeName, mysqlCatalogName, schemaName, tableName))); + mysqlCatalog.asTableCatalog().tableExists(NameIdentifier.of(schemaName, tableName))); Assertions.assertTrue( - postgreSqlCatalog - .asTableCatalog() - .tableExists( - NameIdentifier.of(metalakeName, postgreSqlCatalogName, schemaName, tableName))); + postgreSqlCatalog.asTableCatalog().tableExists(NameIdentifier.of(schemaName, tableName))); } } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java index 631017ec92d..79d5ad906c1 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.ops; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java index 70bb1c70075..f2e4597f6bb 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.utils; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/TestIcebergExceptionMapper.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/TestIcebergExceptionMapper.java index 8c17cbfea40..48e9b5eaf4a 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/TestIcebergExceptionMapper.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/TestIcebergExceptionMapper.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/MemoryMetricsStore.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/MemoryMetricsStore.java index bdca7222b7e..dea1df55dbb 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/MemoryMetricsStore.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/MemoryMetricsStore.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.metrics; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/TestIcebergMetricsManager.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/TestIcebergMetricsManager.java index 1c978e37894..67e017a9792 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/TestIcebergMetricsManager.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/metrics/TestIcebergMetricsManager.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.metrics; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergRestTestUtil.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergRestTestUtil.java index 6cb3a87653d..d35f7f6788f 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergRestTestUtil.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergRestTestUtil.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.rest; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergTestBase.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergTestBase.java index 657c1538423..789fdac6dcf 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergTestBase.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/IcebergTestBase.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.rest; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/TestIcebergConfig.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/TestIcebergConfig.java index 721f637ae8f..07682cdfd0c 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/TestIcebergConfig.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/TestIcebergConfig.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.rest; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/TestIcebergNamespaceOperations.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/TestIcebergNamespaceOperations.java index fec8a10d332..d7b8afbfc49 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/TestIcebergNamespaceOperations.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/TestIcebergNamespaceOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.rest; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/TestIcebergTableOperations.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/TestIcebergTableOperations.java index 3ba448830a4..3b2e939e4fc 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/TestIcebergTableOperations.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/rest/TestIcebergTableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.web.rest; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/resources/log4j2.properties b/catalogs/catalog-lakehouse-iceberg/src/test/resources/log4j2.properties index cb1b9107700..66ed1f47792 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/resources/log4j2.properties +++ b/catalogs/catalog-lakehouse-iceberg/src/test/resources/log4j2.properties @@ -1,6 +1,20 @@ # -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # Set to debug or trace if log4j initialization is failing diff --git a/catalogs/catalog-lakehouse-paimon/build.gradle.kts b/catalogs/catalog-lakehouse-paimon/build.gradle.kts new file mode 100644 index 00000000000..3df919bb398 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/build.gradle.kts @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +description = "catalog-lakehouse-paimon" + +plugins { + `maven-publish` + id("java") + id("idea") +} + +val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString() +val sparkVersion: String = libs.versions.spark34.get() +val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".") +val paimonVersion: String = libs.versions.paimon.get() + +dependencies { + implementation(project(":api")) + implementation(project(":common")) + implementation(project(":core")) + implementation(libs.bundles.paimon) { + exclude("com.sun.jersey") + exclude("javax.servlet") + } + implementation(libs.bundles.log4j) + implementation(libs.commons.lang3) + implementation(libs.guava) + implementation(libs.hadoop2.common) { + exclude("com.github.spotbugs") + exclude("com.sun.jersey") + exclude("javax.servlet") + } + implementation(libs.hadoop2.hdfs) { + exclude("com.sun.jersey") + exclude("javax.servlet") + } + implementation(libs.hadoop2.mapreduce.client.core) { + exclude("com.sun.jersey") + exclude("javax.servlet") + } + + annotationProcessor(libs.lombok) + compileOnly(libs.lombok) + + testImplementation(project(":clients:client-java")) + testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(project(":server")) + testImplementation(project(":server-common")) + testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") { + exclude("org.apache.hadoop") + } + testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") { + exclude("org.apache.avro") + exclude("org.apache.hadoop") + exclude("org.apache.zookeeper") + exclude("io.dropwizard.metrics") + exclude("org.rocksdb") + } + testImplementation("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion") { + exclude("org.apache.hadoop") + } + testImplementation(libs.slf4j.api) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.mysql.driver) + testImplementation(libs.bundles.log4j) + testImplementation(libs.junit.jupiter.params) + testImplementation(libs.testcontainers) + + testRuntimeOnly(libs.junit.jupiter.engine) +} + +tasks { + val runtimeJars by registering(Copy::class) { + from(configurations.runtimeClasspath) + into("build/libs") + } + + val copyCatalogLibs by registering(Copy::class) { + dependsOn("jar", "runtimeJars") + from("build/libs") + into("$rootDir/distribution/package/catalogs/lakehouse-paimon/libs") + } + + val copyCatalogConfig by registering(Copy::class) { + from("src/main/resources") + into("$rootDir/distribution/package/catalogs/lakehouse-paimon/conf") + + include("lakehouse-paimon.conf") + include("core-site.xml.template") + include("hdfs-site.xml.template") + + rename { original -> + if (original.endsWith(".template")) { + original.replace(".template", "") + } else { + original + } + } + + exclude { details -> + details.file.isDirectory() + } + } + + register("copyLibAndConfig", Copy::class) { + dependsOn(copyCatalogLibs, copyCatalogConfig) + } +} + +tasks.test { + val skipUTs = project.hasProperty("skipTests") + if (skipUTs) { + // Only run integration tests + include("**/integration/**") + } + + val skipITs = project.hasProperty("skipITs") + if (skipITs) { + // Exclude integration tests + exclude("**/integration/**") + } else { + dependsOn(tasks.jar) + + doFirst { + environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.12") + } + + val init = project.extra.get("initIntegrationTest") as (Test) -> Unit + init(this) + } +} + +tasks.getByName("generateMetadataFileForMavenJavaPublication") { + dependsOn("runtimeJars") +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/GravitinoPaimonColumn.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/GravitinoPaimonColumn.java new file mode 100644 index 00000000000..1b5c5eb834f --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/GravitinoPaimonColumn.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import static com.datastrato.gravitino.catalog.lakehouse.paimon.utils.TypeUtils.fromPaimonType; +import static com.datastrato.gravitino.catalog.lakehouse.paimon.utils.TypeUtils.toPaimonType; + +import com.datastrato.gravitino.connector.BaseColumn; +import com.datastrato.gravitino.rel.Column; +import java.util.List; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.RowType; + +/** Implementation of {@link Column} that represents a column in the Paimon column. */ +@EqualsAndHashCode(callSuper = true) +public class GravitinoPaimonColumn extends BaseColumn { + + private GravitinoPaimonColumn() {} + + /** + * Converts {@link GravitinoPaimonColumn} instance to inner column. + * + * @param id The id of inner column. + * @return The converted inner column. + */ + public static DataField toPaimonColumn(int id, Column gravitinoColumn) { + DataType paimonType = toPaimonType(gravitinoColumn.dataType()); + DataType paimonTypeWithNullable = + gravitinoColumn.nullable() ? paimonType.nullable() : paimonType.notNull(); + return new DataField( + id, gravitinoColumn.name(), paimonTypeWithNullable, gravitinoColumn.comment()); + } + + /** + * Creates new {@link GravitinoPaimonColumn} instance from Paimon columns. + * + * @param rowType The {@link RowType} instance of Paimon column. + * @return New {@link GravitinoPaimonColumn} instances. + */ + public static List fromPaimonRowType(RowType rowType) { + return rowType.getFields().stream() + .map(GravitinoPaimonColumn::fromPaimonColumn) + .collect(Collectors.toList()); + } + + /** + * Creates a new {@link GravitinoPaimonColumn} instance from inner column. + * + * @param dataField The {@link DataField} instance of inner column. + * @return A new {@link GravitinoPaimonColumn} instance. + */ + public static GravitinoPaimonColumn fromPaimonColumn(DataField dataField) { + return builder() + .withName(dataField.name()) + .withType(fromPaimonType(dataField.type())) + .withComment(dataField.description()) + .withNullable(dataField.type().isNullable()) + .build(); + } + + /** A builder class for constructing {@link GravitinoPaimonColumn} instance. */ + public static class Builder extends BaseColumnBuilder { + + /** Creates a new instance of {@link Builder}. */ + private Builder() {} + + /** + * Internal method to build a {@link GravitinoPaimonColumn} instance using the provided values. + * + * @return A new {@link GravitinoPaimonColumn} instance with the configured values. + */ + @Override + protected GravitinoPaimonColumn internalBuild() { + GravitinoPaimonColumn paimonColumn = new GravitinoPaimonColumn(); + paimonColumn.name = name; + paimonColumn.comment = comment; + paimonColumn.dataType = dataType; + paimonColumn.nullable = nullable; + paimonColumn.autoIncrement = autoIncrement; + paimonColumn.defaultValue = defaultValue == null ? DEFAULT_VALUE_NOT_SET : defaultValue; + return paimonColumn; + } + } + + /** + * Creates a new instance of {@link Builder}. + * + * @return The new instance. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/GravitinoPaimonTable.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/GravitinoPaimonTable.java new file mode 100644 index 00000000000..9d6f26d6679 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/GravitinoPaimonTable.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import static com.datastrato.gravitino.catalog.lakehouse.paimon.GravitinoPaimonColumn.fromPaimonRowType; +import static com.datastrato.gravitino.catalog.lakehouse.paimon.GravitinoPaimonColumn.toPaimonColumn; +import static com.datastrato.gravitino.meta.AuditInfo.EMPTY; + +import com.datastrato.gravitino.connector.BaseTable; +import com.datastrato.gravitino.connector.TableOperations; +import com.google.common.collect.Maps; +import lombok.Getter; +import lombok.ToString; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.DataField; + +/** Implementation of {@link Table} that represents a Paimon Table entity in the Paimon table. */ +@ToString +@Getter +public class GravitinoPaimonTable extends BaseTable { + + private GravitinoPaimonTable() {} + + @Override + protected TableOperations newOps() { + // TODO: Implement this interface when we have the Paimon table operations. + throw new UnsupportedOperationException("PaimonTable does not support TableOperations."); + } + + /** + * Converts {@link GravitinoPaimonTable} instance to Paimon table. + * + * @return The converted Paimon table. + */ + public Schema toPaimonTableSchema() { + Schema.Builder builder = Schema.newBuilder().comment(comment).options(properties); + for (int index = 0; index < columns.length; index++) { + DataField dataField = toPaimonColumn(index, columns[index]); + builder.column(dataField.name(), dataField.type(), dataField.description()); + } + return builder.build(); + } + + /** + * Creates a new {@link GravitinoPaimonTable} instance from Paimon table. + * + * @param table The {@link Table} instance of Paimon table. + * @return A new {@link GravitinoPaimonTable} instance. + */ + public static GravitinoPaimonTable fromPaimonTable(Table table) { + return builder() + .withName(table.name()) + .withColumns(fromPaimonRowType(table.rowType()).toArray(new GravitinoPaimonColumn[0])) + .withComment(table.comment().orElse(null)) + .withProperties(table.options()) + .withAuditInfo(EMPTY) + .build(); + } + + /** A builder class for constructing {@link GravitinoPaimonTable} instance. */ + public static class Builder extends BaseTableBuilder { + + /** Creates a new instance of {@link Builder}. */ + private Builder() {} + + /** + * Internal method to build a {@link GravitinoPaimonTable} instance using the provided values. + * + * @return A new {@link GravitinoPaimonTable} instance with the configured values. + */ + @Override + protected GravitinoPaimonTable internalBuild() { + GravitinoPaimonTable paimonTable = new GravitinoPaimonTable(); + paimonTable.name = name; + paimonTable.comment = comment; + paimonTable.columns = columns; + paimonTable.properties = properties == null ? Maps.newHashMap() : Maps.newHashMap(properties); + paimonTable.auditInfo = auditInfo; + return paimonTable; + } + } + + /** + * Creates a new instance of {@link Builder}. + * + * @return The new instance. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalog.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalog.java new file mode 100644 index 00000000000..7d404bdc880 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalog.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.connector.BaseCatalog; +import com.datastrato.gravitino.connector.CatalogOperations; +import com.datastrato.gravitino.connector.PropertiesMetadata; +import com.datastrato.gravitino.connector.capability.Capability; +import java.util.Map; + +/** Implementation of {@link Catalog} that represents a Paimon catalog in Gravitino. */ +public class PaimonCatalog extends BaseCatalog { + + static final PaimonCatalogPropertiesMetadata CATALOG_PROPERTIES_META = + new PaimonCatalogPropertiesMetadata(); + + static final PaimonSchemaPropertiesMetadata SCHEMA_PROPERTIES_META = + new PaimonSchemaPropertiesMetadata(); + + static final PaimonTablePropertiesMetadata TABLE_PROPERTIES_META = + new PaimonTablePropertiesMetadata(); + + /** @return The short name of the catalog. */ + @Override + public String shortName() { + return "lakehouse-paimon"; + } + + /** + * Creates a new instance of {@link PaimonCatalogOperations} with the provided configuration. + * + * @param config The configuration map for the Paimon catalog operations. + * @return A new instance of {@link PaimonCatalogOperations}. + */ + @Override + protected CatalogOperations newOps(Map config) { + return new PaimonCatalogOperations(); + } + + @Override + public Capability newCapability() { + return new PaimonCatalogCapability(); + } + + @Override + public PropertiesMetadata tablePropertiesMetadata() throws UnsupportedOperationException { + return TABLE_PROPERTIES_META; + } + + @Override + public PropertiesMetadata catalogPropertiesMetadata() throws UnsupportedOperationException { + return CATALOG_PROPERTIES_META; + } + + @Override + public PropertiesMetadata schemaPropertiesMetadata() throws UnsupportedOperationException { + return SCHEMA_PROPERTIES_META; + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalogBackend.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalogBackend.java new file mode 100644 index 00000000000..43bd298fd9c --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalogBackend.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +/** The type of Paimon catalog backend. */ +public enum PaimonCatalogBackend { + FILESYSTEM +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalogCapability.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalogCapability.java new file mode 100644 index 00000000000..f6c6cadca54 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalogCapability.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import com.datastrato.gravitino.connector.capability.Capability; +import com.datastrato.gravitino.connector.capability.CapabilityResult; + +public class PaimonCatalogCapability implements Capability { + + @Override + public CapabilityResult columnDefaultValue() { + // See https://github.com/apache/paimon/pull/1425/files + return CapabilityResult.unsupported( + "Paimon set column default value through table properties instead of column info."); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalogOperations.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalogOperations.java new file mode 100644 index 00000000000..367bfe713d9 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalogOperations.java @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import static com.datastrato.gravitino.catalog.lakehouse.paimon.GravitinoPaimonTable.fromPaimonTable; +import static com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonSchema.fromPaimonProperties; +import static com.datastrato.gravitino.catalog.lakehouse.paimon.utils.TableOpsUtils.checkColumnCapability; +import static com.datastrato.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.SchemaChange; +import com.datastrato.gravitino.catalog.lakehouse.paimon.ops.PaimonCatalogOps; +import com.datastrato.gravitino.connector.CatalogInfo; +import com.datastrato.gravitino.connector.CatalogOperations; +import com.datastrato.gravitino.connector.HasPropertyMetadata; +import com.datastrato.gravitino.connector.SupportsSchemas; +import com.datastrato.gravitino.exceptions.NoSuchCatalogException; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.exceptions.NoSuchTableException; +import com.datastrato.gravitino.exceptions.NonEmptySchemaException; +import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; +import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.TableCatalog; +import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.expressions.distributions.Distribution; +import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.utils.MapUtils; +import com.datastrato.gravitino.utils.PrincipalUtils; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation of {@link CatalogOperations} that represents operations for interacting with the + * Paimon catalog in Gravitino. + */ +public class PaimonCatalogOperations implements CatalogOperations, SupportsSchemas, TableCatalog { + + public static final Logger LOG = LoggerFactory.getLogger(PaimonCatalogOperations.class); + + @VisibleForTesting public PaimonCatalogOps paimonCatalogOps; + + private static final String NO_SUCH_SCHEMA_EXCEPTION = + "Paimon schema (database) %s does not exist."; + private static final String NON_EMPTY_SCHEMA_EXCEPTION = + "Paimon schema (database) %s is not empty. One or more tables exist."; + private static final String SCHEMA_ALREADY_EXISTS_EXCEPTION = + "Paimon schema (database) %s already exists."; + private static final String NO_SUCH_TABLE_EXCEPTION = "Paimon table %s does not exist."; + private static final String TABLE_ALREADY_EXISTS_EXCEPTION = "Paimon table %s already exists."; + + /** + * Initializes the Paimon catalog operations with the provided configuration. + * + * @param conf The configuration map for the Paimon catalog operations. + * @param info The catalog info associated with this operations instance. + * @throws RuntimeException if initialization fails. + */ + @Override + public void initialize( + Map conf, CatalogInfo info, HasPropertyMetadata propertiesMetadata) + throws RuntimeException { + // Key format like gravitino.bypass.a.b + Map prefixMap = MapUtils.getPrefixMap(conf, CATALOG_BYPASS_PREFIX); + + // Hold keys that lie in GRAVITINO_CONFIG_TO_PAIMON + Map gravitinoConfig = + ((PaimonCatalogPropertiesMetadata) propertiesMetadata.catalogPropertiesMetadata()) + .transformProperties(conf); + + Map resultConf = Maps.newHashMap(prefixMap); + resultConf.putAll(gravitinoConfig); + + this.paimonCatalogOps = new PaimonCatalogOps(new PaimonConfig(resultConf)); + } + + /** + * Lists the schemas under the specified namespace. + * + * @param namespace The namespace to list the schemas for. + * @return An array of {@link NameIdentifier} representing the schemas in the namespace. + * @throws NoSuchCatalogException If the provided namespace is invalid or does not exist. + */ + @Override + public NameIdentifier[] listSchemas(Namespace namespace) throws NoSuchCatalogException { + return paimonCatalogOps.listDatabases().stream() + .map(paimonNamespace -> NameIdentifier.of(namespace, paimonNamespace)) + .toArray(NameIdentifier[]::new); + } + + /** + * Creates a new schema with the provided identifier, comment, and metadata. + * + * @param identifier The identifier of the schema to create. + * @param comment The comment for the new schema. + * @param properties The properties for the new schema. + * @return The newly created {@link PaimonSchema} instance. + * @throws NoSuchCatalogException If the provided namespace is invalid or does not exist. + * @throws SchemaAlreadyExistsException If a schema with the same name already exists. + */ + @Override + public PaimonSchema createSchema( + NameIdentifier identifier, String comment, Map properties) + throws NoSuchCatalogException, SchemaAlreadyExistsException { + String currentUser = currentUser(); + PaimonSchema createdSchema = + PaimonSchema.builder() + .withName(identifier.name()) + .withComment(comment) + .withProperties(properties) + .withAuditInfo( + AuditInfo.builder().withCreator(currentUser).withCreateTime(Instant.now()).build()) + .build(); + try { + Map paimonSchemaProperties = createdSchema.toPaimonProperties(); + paimonCatalogOps.createDatabase(identifier.name(), paimonSchemaProperties); + } catch (Catalog.DatabaseAlreadyExistException e) { + throw new SchemaAlreadyExistsException(e, SCHEMA_ALREADY_EXISTS_EXCEPTION, identifier); + } catch (Exception e) { + throw new RuntimeException(e); + } + LOG.info( + "Created Paimon schema (database): {}. Current user: {}. Comment: {}. Metadata: {}.", + identifier, + currentUser, + comment, + properties); + return createdSchema; + } + + /** + * Loads the schema with the provided identifier. + * + * @param identifier The identifier of the schema to load. + * @return The loaded {@link PaimonSchema} representing the schema. + * @throws NoSuchSchemaException If the schema with the provided identifier does not exist. + */ + @Override + public PaimonSchema loadSchema(NameIdentifier identifier) throws NoSuchSchemaException { + Map properties; + try { + properties = paimonCatalogOps.loadDatabase(identifier.name()); + } catch (Catalog.DatabaseNotExistException e) { + throw new NoSuchSchemaException(e, NO_SUCH_SCHEMA_EXCEPTION, identifier); + } + LOG.info("Loaded Paimon schema (database) {}.", identifier); + return fromPaimonProperties(identifier.name(), properties); + } + + /** + * Alters the schema with the provided identifier according to the specified {@link SchemaChange} + * changes. + * + * @param identifier The identifier of the schema to alter. + * @param changes The changes to apply to the schema. + * @return The altered {@link PaimonSchema} instance. + * @throws NoSuchSchemaException If the schema with the provided identifier does not exist. + */ + @Override + public PaimonSchema alterSchema(NameIdentifier identifier, SchemaChange... changes) + throws NoSuchSchemaException { + throw new UnsupportedOperationException("AlterSchema is unsupported now for Paimon Catalog."); + } + + /** + * Drops the schema with the provided identifier. + * + * @param identifier The identifier of the schema to drop. + * @param cascade If set to true, drops all the tables in the schema as well. + * @return true if the schema is dropped successfully, false otherwise. + * @throws NonEmptySchemaException If the schema is not empty and 'cascade' is set to false. + */ + @Override + public boolean dropSchema(NameIdentifier identifier, boolean cascade) + throws NonEmptySchemaException { + try { + paimonCatalogOps.dropDatabase(identifier.name(), cascade); + } catch (Catalog.DatabaseNotExistException e) { + LOG.warn("Paimon schema (database) {} does not exist.", identifier); + return false; + } catch (Catalog.DatabaseNotEmptyException e) { + throw new NonEmptySchemaException(e, NON_EMPTY_SCHEMA_EXCEPTION, identifier); + } catch (Exception e) { + throw new RuntimeException(e); + } + LOG.info("Dropped Paimon schema (database) {}.", identifier); + return true; + } + + /** + * Lists all the tables under the specified namespace. + * + * @param namespace The namespace to list tables for. + * @return An array of {@link NameIdentifier} representing the tables in the namespace. + * @throws NoSuchSchemaException If the schema with the provided namespace does not exist. + */ + @Override + public NameIdentifier[] listTables(Namespace namespace) throws NoSuchSchemaException { + String[] levels = namespace.levels(); + NameIdentifier schemaIdentifier = NameIdentifier.of(levels[levels.length - 1]); + if (!schemaExists(schemaIdentifier)) { + throw new NoSuchSchemaException(NO_SUCH_SCHEMA_EXCEPTION, namespace.toString()); + } + List tables; + try { + tables = paimonCatalogOps.listTables(schemaIdentifier.name()); + } catch (Catalog.DatabaseNotExistException e) { + throw new NoSuchSchemaException(NO_SUCH_SCHEMA_EXCEPTION, namespace.toString()); + } + return tables.stream() + .map( + tableIdentifier -> + NameIdentifier.of(ArrayUtils.add(namespace.levels(), tableIdentifier))) + .toArray(NameIdentifier[]::new); + } + + /** + * Loads the table with the provided identifier. + * + * @param identifier The identifier of the table to load. + * @return The loaded {@link GravitinoPaimonTable} instance representing the table. + * @throws NoSuchTableException If the table with the provided identifier does not exist. + */ + @Override + public GravitinoPaimonTable loadTable(NameIdentifier identifier) throws NoSuchTableException { + Table table; + try { + NameIdentifier tableIdentifier = buildPaimonNameIdentifier(identifier); + table = paimonCatalogOps.loadTable(tableIdentifier.toString()); + } catch (Catalog.TableNotExistException e) { + throw new NoSuchTableException(e, NO_SUCH_TABLE_EXCEPTION, identifier); + } + LOG.info("Loaded Paimon table {}.", identifier); + return fromPaimonTable(table); + } + + /** + * Creates a new table with the provided identifier, comment, and metadata. + * + * @param identifier The identifier of the table to create. + * @param columns The array of columns for the new table. + * @param comment The comment for the new table. + * @param properties The properties for the new table. + * @param partitioning The partitioning for the new table. + * @param indexes The indexes for the new table. + * @return The newly created {@link GravitinoPaimonTable} instance. + * @throws NoSuchSchemaException If the schema with the provided namespace does not exist. + * @throws TableAlreadyExistsException If the table with the same identifier already exists. + */ + @Override + public GravitinoPaimonTable createTable( + NameIdentifier identifier, + Column[] columns, + String comment, + Map properties, + Transform[] partitioning, + Distribution distribution, + SortOrder[] sortOrders, + Index[] indexes) + throws NoSuchSchemaException, TableAlreadyExistsException { + NameIdentifier nameIdentifier = buildPaimonNameIdentifier(identifier); + NameIdentifier schemaIdentifier = NameIdentifier.of(nameIdentifier.namespace().levels()); + if (!schemaExists(schemaIdentifier)) { + throw new NoSuchSchemaException(NO_SUCH_SCHEMA_EXCEPTION, schemaIdentifier); + } + Preconditions.checkArgument( + partitioning == null || partitioning.length == 0, + "Table Partitions are not supported when creating a Paimon table in Gravitino now."); + Preconditions.checkArgument( + sortOrders == null || sortOrders.length == 0, + "Sort orders are not supported for Paimon in Gravitino."); + Preconditions.checkArgument( + indexes == null || indexes.length == 0, + "Indexes are not supported for Paimon in Gravitino."); + Preconditions.checkArgument( + distribution == null || distribution.strategy() == Distributions.NONE.strategy(), + "Distribution is not supported for Paimon in Gravitino now."); + String currentUser = currentUser(); + GravitinoPaimonTable createdTable = + GravitinoPaimonTable.builder() + .withName(identifier.name()) + .withColumns( + Arrays.stream(columns) + .map( + column -> { + checkColumnCapability( + column.name(), column.defaultValue(), column.autoIncrement()); + return GravitinoPaimonColumn.builder() + .withName(column.name()) + .withType(column.dataType()) + .withComment(column.comment()) + .withNullable(column.nullable()) + .withAutoIncrement(column.autoIncrement()) + .withDefaultValue(column.defaultValue()) + .build(); + }) + .toArray(GravitinoPaimonColumn[]::new)) + .withComment(comment) + .withProperties(properties) + .withAuditInfo( + AuditInfo.builder().withCreator(currentUser).withCreateTime(Instant.now()).build()) + .build(); + try { + Schema paimonTableSchema = createdTable.toPaimonTableSchema(); + paimonCatalogOps.createTable(nameIdentifier.toString(), paimonTableSchema); + } catch (Catalog.DatabaseNotExistException e) { + throw new NoSuchSchemaException(e, NO_SUCH_SCHEMA_EXCEPTION, identifier); + } catch (Catalog.TableAlreadyExistException e) { + throw new TableAlreadyExistsException(e, TABLE_ALREADY_EXISTS_EXCEPTION, identifier); + } + LOG.info( + "Created Paimon table: {}. Current user: {}. Comment: {}. Metadata: {}.", + identifier, + currentUser, + comment, + properties); + return createdTable; + } + + /** + * Alters the table with the provided identifier according to the specified {@link TableChange} + * changes. + * + * @param identifier The identifier of the table to alter. + * @param changes The changes to apply to the table. + * @return The altered {@link GravitinoPaimonTable} instance. + * @throws NoSuchTableException If the table with the provided identifier does not exist. + * @throws IllegalArgumentException This exception will not be thrown in this method. + */ + @Override + public GravitinoPaimonTable alterTable(NameIdentifier identifier, TableChange... changes) + throws NoSuchTableException, IllegalArgumentException { + throw new UnsupportedOperationException("alterTable is unsupported now for Paimon Catalog."); + } + + /** + * Drops the table with the provided identifier. + * + * @param identifier The identifier of the table to drop. + * @return true if the table is successfully dropped, false if the table does not exist. + */ + @Override + public boolean dropTable(NameIdentifier identifier) { + try { + NameIdentifier tableIdentifier = buildPaimonNameIdentifier(identifier); + paimonCatalogOps.dropTable(tableIdentifier.toString()); + } catch (Catalog.TableNotExistException e) { + LOG.warn("Paimon table {} does not exist.", identifier); + return false; + } + LOG.info("Dropped Paimon table {}.", identifier); + return true; + } + + /** + * Purges the table with the provided identifier. + * + * @param identifier The identifier of the table to purge. + * @return true if the table is successfully purged, false if the table does not exist. + * @throws UnsupportedOperationException If the table type is EXTERNAL_TABLE, it cannot be purged. + */ + @Override + public boolean purgeTable(NameIdentifier identifier) throws UnsupportedOperationException { + throw new UnsupportedOperationException("purgeTable is unsupported now for Paimon Catalog."); + } + + @Override + public void close() { + if (paimonCatalogOps != null) { + try { + paimonCatalogOps.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + private static String currentUser() { + return PrincipalUtils.getCurrentUserName(); + } + + private NameIdentifier buildPaimonNameIdentifier(NameIdentifier identifier) { + Preconditions.checkArgument( + identifier != null + && identifier.namespace() != null + && identifier.namespace().levels().length > 0, + "Namespace can not be null or empty."); + String[] levels = identifier.namespace().levels(); + return NameIdentifier.of(levels[levels.length - 1], identifier.name()); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java new file mode 100644 index 00000000000..122e851816e --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import static com.datastrato.gravitino.connector.PropertyEntry.enumImmutablePropertyEntry; +import static com.datastrato.gravitino.connector.PropertyEntry.stringOptionalPropertyEntry; +import static com.datastrato.gravitino.connector.PropertyEntry.stringRequiredPropertyEntry; + +import com.datastrato.gravitino.connector.BaseCatalogPropertiesMetadata; +import com.datastrato.gravitino.connector.PropertiesMetadata; +import com.datastrato.gravitino.connector.PropertyEntry; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Implementation of {@link PropertiesMetadata} that represents Paimon catalog properties metadata. + */ +public class PaimonCatalogPropertiesMetadata extends BaseCatalogPropertiesMetadata { + + @VisibleForTesting public static final String GRAVITINO_CATALOG_BACKEND = "catalog-backend"; + @VisibleForTesting public static final String PAIMON_METASTORE = "metastore"; + @VisibleForTesting public static final String WAREHOUSE = "warehouse"; + @VisibleForTesting public static final String URI = "uri"; + + private static final Map> PROPERTIES_METADATA; + private static final Map GRAVITINO_CONFIG_TO_PAIMON = + ImmutableMap.of(GRAVITINO_CATALOG_BACKEND, PAIMON_METASTORE, WAREHOUSE, WAREHOUSE, URI, URI); + + static { + List> propertyEntries = + ImmutableList.of( + enumImmutablePropertyEntry( + GRAVITINO_CATALOG_BACKEND, + "Paimon catalog backend type", + true, + PaimonCatalogBackend.class, + null, + false, + false), + stringRequiredPropertyEntry(WAREHOUSE, "Paimon catalog warehouse config", false, false), + stringOptionalPropertyEntry(URI, "Paimon catalog uri config", false, null, false)); + HashMap> result = Maps.newHashMap(BASIC_CATALOG_PROPERTY_ENTRIES); + result.putAll(Maps.uniqueIndex(propertyEntries, PropertyEntry::getName)); + PROPERTIES_METADATA = ImmutableMap.copyOf(result); + } + + @Override + protected Map> specificPropertyEntries() { + return PROPERTIES_METADATA; + } + + protected Map transformProperties(Map properties) { + Map gravitinoConfig = Maps.newHashMap(); + properties.forEach( + (key, value) -> { + if (GRAVITINO_CONFIG_TO_PAIMON.containsKey(key)) { + gravitinoConfig.put(GRAVITINO_CONFIG_TO_PAIMON.get(key), value); + } + }); + return gravitinoConfig; + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonConfig.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonConfig.java new file mode 100644 index 00000000000..459afb2fae1 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonConfig.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.config.ConfigBuilder; +import com.datastrato.gravitino.config.ConfigConstants; +import com.datastrato.gravitino.config.ConfigEntry; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.paimon.options.CatalogOptions; + +public class PaimonConfig extends Config { + + public static final ConfigEntry CATALOG_BACKEND = + new ConfigBuilder(PaimonCatalogPropertiesMetadata.PAIMON_METASTORE) + .doc(CatalogOptions.METASTORE.description().toString()) + .version(ConfigConstants.VERSION_0_6_0) + .stringConf() + .createWithDefault(CatalogOptions.METASTORE.defaultValue()); + + public static final ConfigEntry CATALOG_WAREHOUSE = + new ConfigBuilder(PaimonCatalogPropertiesMetadata.WAREHOUSE) + .doc(CatalogOptions.WAREHOUSE.description().toString()) + .version(ConfigConstants.VERSION_0_6_0) + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + public static final ConfigEntry CATALOG_URI = + new ConfigBuilder(PaimonCatalogPropertiesMetadata.URI) + .doc(CatalogOptions.URI.description().toString()) + .version(ConfigConstants.VERSION_0_6_0) + .stringConf() + .create(); + + public PaimonConfig() { + super(false); + } + + public PaimonConfig(Map properties) { + super(false); + loadFromMap(properties, k -> true); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonSchema.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonSchema.java new file mode 100644 index 00000000000..1c8b4576c55 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonSchema.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import static com.datastrato.gravitino.meta.AuditInfo.EMPTY; + +import com.datastrato.gravitino.Schema; +import com.datastrato.gravitino.connector.BaseSchema; +import java.util.Map; +import java.util.Optional; +import lombok.ToString; + +/** + * Implementation of {@link Schema} that represents a Paimon Schema (Database) entity in the Paimon + * schema. + */ +@ToString +public class PaimonSchema extends BaseSchema { + + private PaimonSchema() {} + + /** + * Converts {@link PaimonSchema} instance to inner schema. + * + * @return The converted inner schema. + */ + public Map toPaimonProperties() { + return properties; + } + + /** + * Creates a new {@link PaimonSchema} instance from inner schema. + * + * @param name The name of inner schema. + * @param properties The properties of inner schema. + * @return A new {@link PaimonSchema} instance. + */ + public static PaimonSchema fromPaimonProperties(String name, Map properties) { + return builder() + .withName(name) + .withComment( + Optional.of(properties) + .map(map -> map.get(PaimonSchemaPropertiesMetadata.COMMENT)) + .orElse(null)) + .withProperties(properties) + .withAuditInfo(EMPTY) + .build(); + } + + /** A builder class for constructing {@link PaimonSchema} instance. */ + public static class Builder extends BaseSchemaBuilder { + + /** Creates a new instance of {@link Builder}. */ + private Builder() {} + + /** + * Internal method to build a {@link PaimonSchema} instance using the provided values. + * + * @return A new {@link PaimonSchema} instance with the configured values. + */ + @Override + protected PaimonSchema internalBuild() { + PaimonSchema paimonSchema = new PaimonSchema(); + paimonSchema.name = name; + paimonSchema.comment = + comment == null + ? (properties == null ? null : properties.get(PaimonSchemaPropertiesMetadata.COMMENT)) + : comment; + paimonSchema.properties = properties; + paimonSchema.auditInfo = auditInfo; + return paimonSchema; + } + } + + /** + * Creates a new instance of {@link Builder}. + * + * @return The new instance. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonSchemaPropertiesMetadata.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonSchemaPropertiesMetadata.java new file mode 100644 index 00000000000..efebf874ba0 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonSchemaPropertiesMetadata.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import static com.datastrato.gravitino.connector.PropertyEntry.stringReservedPropertyEntry; + +import com.datastrato.gravitino.connector.BasePropertiesMetadata; +import com.datastrato.gravitino.connector.PropertiesMetadata; +import com.datastrato.gravitino.connector.PropertyEntry; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import java.util.List; +import java.util.Map; + +/** + * Implementation of {@link PropertiesMetadata} that represents Paimon schema properties metadata. + */ +public class PaimonSchemaPropertiesMetadata extends BasePropertiesMetadata { + + public static final String COMMENT = "comment"; + + private static final Map> PROPERTIES_METADATA; + + static { + List> propertyEntries = + ImmutableList.of(stringReservedPropertyEntry(COMMENT, "Schema comment", true)); + PROPERTIES_METADATA = Maps.uniqueIndex(propertyEntries, PropertyEntry::getName); + } + + @Override + protected Map> specificPropertyEntries() { + return PROPERTIES_METADATA; + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonTablePropertiesMetadata.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonTablePropertiesMetadata.java new file mode 100644 index 00000000000..c94a43358db --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/PaimonTablePropertiesMetadata.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import static com.datastrato.gravitino.connector.PropertyEntry.stringReservedPropertyEntry; + +import com.datastrato.gravitino.connector.BasePropertiesMetadata; +import com.datastrato.gravitino.connector.PropertiesMetadata; +import com.datastrato.gravitino.connector.PropertyEntry; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import java.util.List; +import java.util.Map; + +/** + * Implementation of {@link PropertiesMetadata} that represents Paimon table properties metadata. + */ +public class PaimonTablePropertiesMetadata extends BasePropertiesMetadata { + + public static final String COMMENT = "comment"; + public static final String CREATOR = "creator"; + + private static final Map> PROPERTIES_METADATA; + + static { + List> propertyEntries = + ImmutableList.of( + stringReservedPropertyEntry(COMMENT, "The table comment", true), + stringReservedPropertyEntry(CREATOR, "The table creator", false)); + PROPERTIES_METADATA = Maps.uniqueIndex(propertyEntries, PropertyEntry::getName); + } + + @Override + protected Map> specificPropertyEntries() { + return PROPERTIES_METADATA; + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/ops/PaimonCatalogOps.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/ops/PaimonCatalogOps.java new file mode 100644 index 00000000000..586998cb9f1 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/ops/PaimonCatalogOps.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon.ops; + +import static com.datastrato.gravitino.catalog.lakehouse.paimon.utils.CatalogUtils.loadCatalogBackend; + +import com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonConfig; +import java.util.List; +import java.util.Map; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Catalog.DatabaseAlreadyExistException; +import org.apache.paimon.catalog.Catalog.DatabaseNotEmptyException; +import org.apache.paimon.catalog.Catalog.DatabaseNotExistException; +import org.apache.paimon.catalog.Catalog.TableNotExistException; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; + +/** Table operation proxy that handles table operations of an underlying Paimon catalog. */ +public class PaimonCatalogOps implements AutoCloseable { + + protected Catalog catalog; + + public PaimonCatalogOps(PaimonConfig paimonConfig) { + catalog = loadCatalogBackend(paimonConfig); + } + + @Override + public void close() throws Exception { + if (catalog != null) { + catalog.close(); + } + } + + public List listDatabases() { + return catalog.listDatabases(); + } + + public Map loadDatabase(String databaseName) throws DatabaseNotExistException { + return catalog.loadDatabaseProperties(databaseName); + } + + public void createDatabase(String databaseName, Map properties) + throws DatabaseAlreadyExistException { + catalog.createDatabase(databaseName, false, properties); + } + + public void dropDatabase(String databaseName, boolean cascade) + throws DatabaseNotExistException, DatabaseNotEmptyException { + catalog.dropDatabase(databaseName, false, cascade); + } + + public List listTables(String databaseName) throws DatabaseNotExistException { + return catalog.listTables(databaseName); + } + + public Table loadTable(String tableName) throws TableNotExistException { + return catalog.getTable(tableIdentifier(tableName)); + } + + public void createTable(String tablename, Schema schema) + throws Catalog.TableAlreadyExistException, DatabaseNotExistException { + catalog.createTable(tableIdentifier(tablename), schema, false); + } + + public void dropTable(String tableName) throws TableNotExistException { + catalog.dropTable(tableIdentifier(tableName), false); + } + + private Identifier tableIdentifier(String tableName) { + return Identifier.fromString(tableName); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/CatalogUtils.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/CatalogUtils.java new file mode 100644 index 00000000000..d28dbaa12a5 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/CatalogUtils.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon.utils; + +import static com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonConfig.CATALOG_BACKEND; +import static com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonConfig.CATALOG_URI; +import static com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonConfig.CATALOG_WAREHOUSE; + +import com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonCatalogBackend; +import com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonConfig; +import com.google.common.base.Preconditions; +import org.apache.commons.lang3.StringUtils; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.options.Options; + +/** Utilities of {@link Catalog} to support catalog management. */ +public class CatalogUtils { + + private CatalogUtils() {} + + /** + * Loads {@link Catalog} instance with given {@link PaimonConfig}. + * + * @param paimonConfig The Paimon configuration. + * @return The {@link Catalog} instance of catalog backend. + */ + public static Catalog loadCatalogBackend(PaimonConfig paimonConfig) { + String metastore = paimonConfig.get(CATALOG_BACKEND); + Preconditions.checkArgument( + StringUtils.isNotBlank(metastore), "Paimon Catalog metastore can not be null or empty."); + String warehouse = paimonConfig.get(CATALOG_WAREHOUSE); + Preconditions.checkArgument( + StringUtils.isNotBlank(warehouse), "Paimon Catalog warehouse can not be null or empty."); + if (!PaimonCatalogBackend.FILESYSTEM.name().equalsIgnoreCase(metastore)) { + String uri = paimonConfig.get(CATALOG_URI); + Preconditions.checkArgument( + StringUtils.isNotBlank(uri), + String.format("Paimon Catalog uri can not be null or empty for %s.", metastore)); + } + CatalogContext catalogContext = + CatalogContext.create(Options.fromMap(paimonConfig.getAllConfig())); + return CatalogFactory.createCatalog(catalogContext); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TableOpsUtils.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TableOpsUtils.java new file mode 100644 index 00000000000..83ee694cd60 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TableOpsUtils.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon.utils; + +import com.datastrato.gravitino.catalog.lakehouse.paimon.ops.PaimonCatalogOps; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.expressions.Expression; +import com.google.common.base.Preconditions; + +/** Utilities of {@link PaimonCatalogOps} to support table operation. */ +public class TableOpsUtils { + + public static void checkColumnCapability( + String fieldName, Expression defaultValue, boolean autoIncrement) { + checkColumnDefaultValue(fieldName, defaultValue); + checkColumnAutoIncrement(fieldName, autoIncrement); + } + + private static void checkColumnDefaultValue(String fieldName, Expression defaultValue) { + Preconditions.checkArgument( + defaultValue.equals(Column.DEFAULT_VALUE_NOT_SET), + String.format( + "Paimon set column default value through table properties instead of column info. Illegal column: %s.", + fieldName)); + } + + private static void checkColumnAutoIncrement(String fieldName, boolean autoIncrement) { + Preconditions.checkArgument( + !autoIncrement, + String.format( + "Paimon does not support auto increment column. Illegal column: %s.", fieldName)); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TypeUtils.java b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TypeUtils.java new file mode 100644 index 00000000000..9022135c778 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TypeUtils.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.datastrato.gravitino.catalog.lakehouse.paimon.utils; + +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; +import java.util.Arrays; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BooleanType; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeDefaultVisitor; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.DateType; +import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.DoubleType; +import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.SmallIntType; +import org.apache.paimon.types.TimeType; +import org.apache.paimon.types.TimestampType; +import org.apache.paimon.types.TinyIntType; +import org.apache.paimon.types.VarCharType; + +// Referred to org/apache/paimon/spark/SparkTypeUtils.java +/** Utilities of {@link Type} to support type conversion. */ +public class TypeUtils { + + private TypeUtils() {} + + /** + * Convert Paimon {@link DataType} data type to Gravitino {@link Type} data type. + * + * @param dataType Paimon {@link DataType} data type. + * @return Gravitino {@link Type} data type. + */ + public static Type fromPaimonType(DataType dataType) { + return dataType.accept(PaimonToGravitinoTypeVisitor.INSTANCE); + } + + /** + * Convert Gravitino {@link Type} data type to Paimon {@link DataType} data type. + * + * @param type Gravitino {@link Type} data type. + * @return Paimon {@link DataType} data type. + */ + public static DataType toPaimonType(Type type) { + return GravitinoToPaimonTypeVisitor.visit(type); + } + + private static class PaimonToGravitinoTypeVisitor extends DataTypeDefaultVisitor { + + private static final PaimonToGravitinoTypeVisitor INSTANCE = new PaimonToGravitinoTypeVisitor(); + + @Override + public Type visit(VarCharType varCharType) { + if (varCharType.getLength() == Integer.MAX_VALUE) { + return Types.StringType.get(); + } else { + return Types.VarCharType.of(varCharType.getLength()); + } + } + + @Override + public Type visit(BooleanType booleanType) { + return Types.BooleanType.get(); + } + + @Override + public Type visit(BinaryType binaryType) { + return Types.BinaryType.get(); + } + + @Override + public Type visit(DecimalType decimalType) { + return Types.DecimalType.of(decimalType.getPrecision(), decimalType.getScale()); + } + + @Override + public Type visit(TinyIntType tinyIntType) { + return Types.ByteType.get(); + } + + @Override + public Type visit(SmallIntType smallIntType) { + return Types.ShortType.get(); + } + + @Override + public Type visit(IntType intType) { + return Types.IntegerType.get(); + } + + @Override + public Type visit(BigIntType bigIntType) { + return Types.LongType.get(); + } + + @Override + public Type visit(FloatType floatType) { + return Types.FloatType.get(); + } + + @Override + public Type visit(DoubleType doubleType) { + return Types.DoubleType.get(); + } + + @Override + public Type visit(DateType dateType) { + return Types.DateType.get(); + } + + @Override + public Type visit(TimeType timeType) { + return Types.TimeType.get(); + } + + @Override + public Type visit(TimestampType timestampType) { + return Types.TimestampType.withoutTimeZone(); + } + + @Override + public Type visit(LocalZonedTimestampType localZonedTimestampType) { + return Types.TimestampType.withTimeZone(); + } + + @Override + public Type visit(ArrayType arrayType) { + return Types.ListType.of( + arrayType.getElementType().accept(this), arrayType.getElementType().isNullable()); + } + + @Override + public Type visit(MultisetType multisetType) { + // Unlike a Java Set, MultisetType allows for multiple instances for each of its + // elements with a common subtype. And a conversion is possible through a map + // that assigns each value to an integer to represent the multiplicity of the values. + // For example, a `MULTISET` is converted to a `MAP`, the key of the + // map represents the elements of the Multiset and the value represents the multiplicity of + // the elements in the Multiset. + return Types.MapType.of( + multisetType.getElementType().accept(this), Types.IntegerType.get(), false); + } + + @Override + public Type visit(MapType mapType) { + return Types.MapType.of( + mapType.getKeyType().accept(this), + mapType.getValueType().accept(this), + mapType.getValueType().isNullable()); + } + + @Override + public Type visit(RowType rowType) { + return Types.StructType.of( + rowType.getFields().stream() + .map( + field -> + Types.StructType.Field.of( + field.name(), + field.type().accept(this), + field.type().isNullable(), + field.description())) + .toArray(Types.StructType.Field[]::new)); + } + + @Override + protected Type defaultMethod(DataType dataType) { + return Types.UnparsedType.of(dataType.asSQLString()); + } + } + + private static class GravitinoToPaimonTypeVisitor { + + public static DataType visit(Type type) { + switch (type.name()) { + case BOOLEAN: + return DataTypes.BOOLEAN(); + case BYTE: + return DataTypes.TINYINT(); + case SHORT: + return DataTypes.SMALLINT(); + case INTEGER: + return DataTypes.INT(); + case LONG: + return DataTypes.BIGINT(); + case FLOAT: + return DataTypes.FLOAT(); + case DOUBLE: + return DataTypes.DOUBLE(); + case DECIMAL: + Types.DecimalType decimalType = (Types.DecimalType) type; + return DataTypes.DECIMAL(decimalType.precision(), decimalType.scale()); + case BINARY: + return DataTypes.BINARY(BinaryType.MAX_LENGTH); + case STRING: + return DataTypes.STRING(); + case VARCHAR: + return DataTypes.VARCHAR(((Types.VarCharType) type).length()); + case DATE: + return DataTypes.DATE(); + case TIME: + return DataTypes.TIME(); + case TIMESTAMP: + return ((Types.TimestampType) type).hasTimeZone() + ? DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE() + : DataTypes.TIMESTAMP(); + case LIST: + Types.ListType listType = (Types.ListType) type; + return DataTypes.ARRAY(visit(listType.elementType())); + case MAP: + Types.MapType mapType = (Types.MapType) type; + return DataTypes.MAP(visit(mapType.keyType()), visit(mapType.valueType())); + case STRUCT: + RowType.Builder builder = RowType.builder(); + Arrays.stream(((Types.StructType) type).fields()) + .forEach( + field -> { + DataType dataType = GravitinoToPaimonTypeVisitor.visit(field.type()); + DataType dataTypeWithNullable = + field.nullable() ? dataType.nullable() : dataType.notNull(); + builder.field(field.name(), dataTypeWithNullable, field.comment()); + }); + return builder.build(); + default: + throw new UnsupportedOperationException( + String.format( + "Paimon does not support Gravitino %s data type.", type.simpleString())); + } + } + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider b/catalogs/catalog-lakehouse-paimon/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider new file mode 100644 index 00000000000..2cb572cd3c6 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonCatalog diff --git a/catalogs/catalog-lakehouse-paimon/src/main/resources/core-site.xml.template b/catalogs/catalog-lakehouse-paimon/src/main/resources/core-site.xml.template new file mode 100644 index 00000000000..efa7db5adb6 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/resources/core-site.xml.template @@ -0,0 +1,21 @@ + + + + diff --git a/catalogs/catalog-lakehouse-paimon/src/main/resources/hdfs-site.xml.template b/catalogs/catalog-lakehouse-paimon/src/main/resources/hdfs-site.xml.template new file mode 100644 index 00000000000..a47ec72adb9 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/resources/hdfs-site.xml.template @@ -0,0 +1,20 @@ + + + diff --git a/catalogs/catalog-lakehouse-paimon/src/main/resources/lakehouse-paimon.conf b/catalogs/catalog-lakehouse-paimon/src/main/resources/lakehouse-paimon.conf new file mode 100644 index 00000000000..4ec5bd974e6 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/main/resources/lakehouse-paimon.conf @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +## This file holds common configurations for lakehouse-paimon catalog. The format of the key is +## 'gravitino.bypass.{paimon-inner-config-key}' and `paimon-inner-config-key` is the +## actual key that pass to lakehouse-paimon catalog. diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java new file mode 100644 index 00000000000..eba62d4dedc --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java @@ -0,0 +1,358 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import static com.datastrato.gravitino.catalog.lakehouse.paimon.GravitinoPaimonColumn.fromPaimonColumn; +import static com.datastrato.gravitino.catalog.lakehouse.paimon.TestPaimonCatalog.PAIMON_PROPERTIES_METADATA; +import static com.datastrato.gravitino.catalog.lakehouse.paimon.utils.TableOpsUtils.checkColumnCapability; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.PropertiesMetadataHelpers; +import com.datastrato.gravitino.connector.PropertiesMetadata; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.CatalogEntity; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.TableCatalog; +import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.types.Types; +import com.google.common.collect.Maps; +import java.time.Instant; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.DateType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class TestGravitinoPaimonTable { + + private static final String META_LAKE_NAME = "metalake"; + + private static final String PAIMON_CATALOG_NAME = "test_catalog"; + private static final String PAIMON_SCHEMA_NAME = "test_schema"; + private static final String PAIMON_COMMENT = "test_comment"; + private static PaimonCatalog paimonCatalog; + private static PaimonCatalogOperations paimonCatalogOperations; + private static PaimonSchema paimonSchema; + private static final NameIdentifier schemaIdent = + NameIdentifier.of(META_LAKE_NAME, PAIMON_CATALOG_NAME, PAIMON_SCHEMA_NAME); + + @BeforeAll + static void setup() { + initPaimonCatalog(); + initPaimonSchema(); + } + + @AfterEach + void resetSchema() { + NameIdentifier[] nameIdentifiers = + paimonCatalogOperations.listTables( + Namespace.of(ArrayUtils.add(schemaIdent.namespace().levels(), schemaIdent.name()))); + if (ArrayUtils.isNotEmpty(nameIdentifiers)) { + Arrays.stream(nameIdentifiers) + .map( + nameIdentifier -> { + String[] levels = nameIdentifier.namespace().levels(); + return NameIdentifier.of( + Namespace.of(levels[levels.length - 1]), nameIdentifier.name()); + }) + .forEach(nameIdentifier -> paimonCatalogOperations.dropTable(nameIdentifier)); + } + paimonCatalogOperations.dropSchema(schemaIdent, false); + initPaimonSchema(); + } + + @AfterAll + static void cleanUp() { + paimonCatalogOperations.dropSchema(schemaIdent, true); + } + + private static CatalogEntity createDefaultCatalogEntity() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("testPaimonUser").withCreateTime(Instant.now()).build(); + + CatalogEntity entity = + CatalogEntity.builder() + .withId(1L) + .withName(PAIMON_CATALOG_NAME) + .withNamespace(Namespace.of(META_LAKE_NAME)) + .withType(PaimonCatalog.Type.RELATIONAL) + .withProvider("lakehouse-paimon") + .withAuditInfo(auditInfo) + .build(); + return entity; + } + + @Test + void testCreatePaimonTable() { + String paimonTableName = "test_paimon_table"; + NameIdentifier tableIdentifier = NameIdentifier.of(paimonSchema.name(), paimonTableName); + Map properties = Maps.newHashMap(); + properties.put("key1", "val1"); + properties.put("key2", "val2"); + + GravitinoPaimonColumn col1 = + fromPaimonColumn(new DataField(0, "col_1", DataTypes.INT().nullable(), PAIMON_COMMENT)); + GravitinoPaimonColumn col2 = + fromPaimonColumn(new DataField(1, "col_2", DataTypes.DATE().notNull(), PAIMON_COMMENT)); + RowType rowTypeInside = + RowType.builder() + .field("integer_field_inside", DataTypes.INT().notNull()) + .field("string_field_inside", DataTypes.STRING().notNull()) + .build(); + RowType rowType = + RowType.builder() + .field("integer_field", DataTypes.INT().notNull()) + .field("string_field", DataTypes.STRING().notNull(), "string field") + .field("struct_field", rowTypeInside.nullable(), "struct field") + .build(); + GravitinoPaimonColumn col3 = + fromPaimonColumn(new DataField(2, "col_3", rowType.notNull(), PAIMON_COMMENT)); + + Column[] columns = new Column[] {col1, col2, col3}; + Table table = + paimonCatalogOperations.createTable( + tableIdentifier, + columns, + PAIMON_COMMENT, + properties, + new Transform[0], + Distributions.NONE, + new SortOrder[0]); + + Assertions.assertEquals(tableIdentifier.name(), table.name()); + Assertions.assertEquals(PAIMON_COMMENT, table.comment()); + Assertions.assertEquals("val1", table.properties().get("key1")); + Assertions.assertEquals("val2", table.properties().get("key2")); + + Table loadedTable = paimonCatalogOperations.loadTable(tableIdentifier); + + Assertions.assertEquals("val1", loadedTable.properties().get("key1")); + Assertions.assertEquals("val2", loadedTable.properties().get("key2")); + Assertions.assertTrue(loadedTable.columns()[0].nullable()); + Assertions.assertFalse(loadedTable.columns()[1].nullable()); + Assertions.assertFalse(loadedTable.columns()[2].nullable()); + + Assertions.assertTrue(paimonCatalogOperations.tableExists(tableIdentifier)); + NameIdentifier[] tableIdents = paimonCatalogOperations.listTables(tableIdentifier.namespace()); + Assertions.assertTrue(Arrays.asList(tableIdents).contains(tableIdentifier)); + + // Test exception + TableCatalog tableCatalog = paimonCatalogOperations; + Throwable exception = + Assertions.assertThrows( + TableAlreadyExistsException.class, + () -> + tableCatalog.createTable( + tableIdentifier, + columns, + PAIMON_COMMENT, + properties, + new Transform[0], + Distributions.NONE, + new SortOrder[0])); + Assertions.assertTrue( + exception + .getMessage() + .contains(String.format("Paimon table %s already exists", tableIdentifier))); + } + + @Test + void testDropPaimonTable() { + NameIdentifier tableIdentifier = NameIdentifier.of(paimonSchema.name(), genRandomName()); + Map properties = Maps.newHashMap(); + properties.put("key1", "val1"); + properties.put("key2", "val2"); + + GravitinoPaimonColumn col1 = + fromPaimonColumn(new DataField(0, "col_1", DataTypes.INT().nullable(), PAIMON_COMMENT)); + GravitinoPaimonColumn col2 = + fromPaimonColumn(new DataField(1, "col_2", DataTypes.DATE().nullable(), PAIMON_COMMENT)); + Column[] columns = new Column[] {col1, col2}; + + paimonCatalogOperations.createTable( + tableIdentifier, + columns, + PAIMON_COMMENT, + properties, + new Transform[0], + Distributions.NONE, + new SortOrder[0]); + + Assertions.assertTrue(paimonCatalogOperations.tableExists(tableIdentifier)); + paimonCatalogOperations.dropTable(tableIdentifier); + Assertions.assertFalse(paimonCatalogOperations.tableExists(tableIdentifier)); + } + + @Test + void testListTableException() { + Namespace tableNs = Namespace.of("metalake", paimonCatalog.name(), "not_exist_db"); + TableCatalog tableCatalog = paimonCatalogOperations; + Throwable exception = + Assertions.assertThrows( + NoSuchSchemaException.class, () -> tableCatalog.listTables(tableNs)); + Assertions.assertTrue( + exception + .getMessage() + .contains( + String.format("Paimon schema (database) %s does not exist", tableNs.toString()))); + } + + @Test + void testTableProperty() { + CatalogEntity entity = createDefaultCatalogEntity(); + try (PaimonCatalogOperations ops = new PaimonCatalogOperations()) { + ops.initialize( + initBackendCatalogProperties(), entity.toCatalogInfo(), PAIMON_PROPERTIES_METADATA); + Map map = Maps.newHashMap(); + map.put(PaimonTablePropertiesMetadata.COMMENT, "test"); + map.put(PaimonTablePropertiesMetadata.CREATOR, "test"); + for (Map.Entry entry : map.entrySet()) { + HashMap properties = + new HashMap() { + { + put(entry.getKey(), entry.getValue()); + } + }; + PropertiesMetadata metadata = paimonCatalog.tablePropertiesMetadata(); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> PropertiesMetadataHelpers.validatePropertyForCreate(metadata, properties)); + } + + map = Maps.newHashMap(); + map.put("key1", "val1"); + map.put("key2", "val2"); + for (Map.Entry entry : map.entrySet()) { + HashMap properties = + new HashMap() { + { + put(entry.getKey(), entry.getValue()); + } + }; + PropertiesMetadata metadata = paimonCatalog.tablePropertiesMetadata(); + Assertions.assertDoesNotThrow( + () -> { + PropertiesMetadataHelpers.validatePropertyForCreate(metadata, properties); + }); + } + } + } + + @Test + void testGravitinoToPaimonTable() { + Column[] columns = createColumns(); + NameIdentifier identifier = NameIdentifier.of("test_schema", "test_table"); + Map properties = Maps.newHashMap(); + properties.put("key1", "val1"); + + GravitinoPaimonTable gravitinoPaimonTable = + GravitinoPaimonTable.builder() + .withName(identifier.name()) + .withColumns( + Arrays.stream(columns) + .map( + column -> { + checkColumnCapability( + column.name(), column.defaultValue(), column.autoIncrement()); + return GravitinoPaimonColumn.builder() + .withName(column.name()) + .withType(column.dataType()) + .withComment(column.comment()) + .withNullable(column.nullable()) + .withAutoIncrement(column.autoIncrement()) + .withDefaultValue(column.defaultValue()) + .build(); + }) + .toArray(GravitinoPaimonColumn[]::new)) + .withComment("test_table_comment") + .withProperties(properties) + .build(); + Schema paimonTableSchema = gravitinoPaimonTable.toPaimonTableSchema(); + Assertions.assertEquals(gravitinoPaimonTable.comment(), gravitinoPaimonTable.comment()); + Assertions.assertEquals(gravitinoPaimonTable.properties(), paimonTableSchema.options()); + Assertions.assertEquals( + gravitinoPaimonTable.columns().length, paimonTableSchema.fields().size()); + Assertions.assertEquals(3, paimonTableSchema.fields().size()); + for (int i = 0; i < gravitinoPaimonTable.columns().length; i++) { + Column column = gravitinoPaimonTable.columns()[i]; + DataField dataField = paimonTableSchema.fields().get(i); + Assertions.assertEquals(column.name(), dataField.name()); + Assertions.assertEquals(column.comment(), dataField.description()); + } + Assertions.assertEquals(new IntType().nullable(), paimonTableSchema.fields().get(0).type()); + Assertions.assertEquals(new DateType().nullable(), paimonTableSchema.fields().get(1).type()); + Assertions.assertEquals( + new VarCharType(Integer.MAX_VALUE).nullable(), paimonTableSchema.fields().get(2).type()); + } + + private static String genRandomName() { + return UUID.randomUUID().toString().replace("-", ""); + } + + private static Map initBackendCatalogProperties() { + Map conf = Maps.newHashMap(); + conf.put(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND, "filesystem"); + conf.put(PaimonCatalogPropertiesMetadata.WAREHOUSE, "/tmp/paimon_catalog_warehouse"); + return conf; + } + + private static void initPaimonCatalog() { + CatalogEntity entity = createDefaultCatalogEntity(); + + Map conf = initBackendCatalogProperties(); + paimonCatalog = new PaimonCatalog().withCatalogConf(conf).withCatalogEntity(entity); + paimonCatalogOperations = (PaimonCatalogOperations) paimonCatalog.ops(); + } + + private static void initPaimonSchema() { + Map properties = Maps.newHashMap(); + properties.put("key1", "val1"); + properties.put("key2", "val2"); + + if (paimonCatalogOperations.schemaExists(schemaIdent)) { + paimonCatalogOperations.dropSchema(schemaIdent, true); + } + paimonSchema = paimonCatalogOperations.createSchema(schemaIdent, PAIMON_COMMENT, properties); + } + + private static Column[] createColumns() { + Column col1 = Column.of("col1", Types.IntegerType.get(), "col_1_comment"); + Column col2 = Column.of("col2", Types.DateType.get(), "col_2_comment"); + Column col3 = Column.of("col3", Types.StringType.get(), "col_3_comment"); + return new Column[] {col1, col2, col3}; + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java new file mode 100644 index 00000000000..fb8cb2bd633 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import static com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonCatalog.CATALOG_PROPERTIES_META; +import static com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonCatalog.SCHEMA_PROPERTIES_META; +import static com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonCatalog.TABLE_PROPERTIES_META; + +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.PropertiesMetadataHelpers; +import com.datastrato.gravitino.catalog.lakehouse.paimon.ops.PaimonCatalogOps; +import com.datastrato.gravitino.connector.CatalogOperations; +import com.datastrato.gravitino.connector.HasPropertyMetadata; +import com.datastrato.gravitino.connector.PropertiesMetadata; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.CatalogEntity; +import com.google.common.collect.Maps; +import java.io.File; +import java.io.IOException; +import java.time.Instant; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestPaimonCatalog { + + static final HasPropertyMetadata PAIMON_PROPERTIES_METADATA = + new HasPropertyMetadata() { + + @Override + public PropertiesMetadata tablePropertiesMetadata() throws UnsupportedOperationException { + return TABLE_PROPERTIES_META; + } + + @Override + public PropertiesMetadata catalogPropertiesMetadata() throws UnsupportedOperationException { + return CATALOG_PROPERTIES_META; + } + + @Override + public PropertiesMetadata schemaPropertiesMetadata() throws UnsupportedOperationException { + return SCHEMA_PROPERTIES_META; + } + + @Override + public PropertiesMetadata filesetPropertiesMetadata() throws UnsupportedOperationException { + throw new UnsupportedOperationException("Fileset properties are not supported"); + } + + @Override + public PropertiesMetadata topicPropertiesMetadata() throws UnsupportedOperationException { + throw new UnsupportedOperationException("Topic properties are not supported"); + } + }; + + private String tempDir = + String.join(File.separator, System.getProperty("java.io.tmpdir"), "paimon_catalog_warehouse"); + + @Test + public void testCatalogOperation() { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + + CatalogEntity entity = + CatalogEntity.builder() + .withId(1L) + .withName("catalog") + .withNamespace(Namespace.of("metalake")) + .withType(PaimonCatalog.Type.RELATIONAL) + .withProvider("lakehouse-paimon") + .withAuditInfo(auditInfo) + .build(); + + Map conf = Maps.newHashMap(); + conf.put(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND, "filesystem"); + conf.put(PaimonCatalogPropertiesMetadata.WAREHOUSE, tempDir); + PaimonCatalog paimonCatalog = + new PaimonCatalog().withCatalogConf(conf).withCatalogEntity(entity); + CatalogOperations catalogOperations = paimonCatalog.ops(); + Assertions.assertInstanceOf(PaimonCatalogOperations.class, catalogOperations); + + PaimonCatalogOperations paimonCatalogOperations = (PaimonCatalogOperations) catalogOperations; + PaimonCatalogOps paimonCatalogOps = paimonCatalogOperations.paimonCatalogOps; + Assertions.assertEquals( + paimonCatalogOperations.listSchemas(Namespace.empty()).length, + paimonCatalogOps.listDatabases().size()); + } + + @Test + void testCatalogProperty() throws IOException { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + + CatalogEntity entity = + CatalogEntity.builder() + .withId(1L) + .withName("catalog") + .withNamespace(Namespace.of("metalake")) + .withType(PaimonCatalog.Type.RELATIONAL) + .withProvider("lakehouse-paimon") + .withAuditInfo(auditInfo) + .build(); + + Map conf = Maps.newHashMap(); + conf.put(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND, "filesystem"); + conf.put(PaimonCatalogPropertiesMetadata.WAREHOUSE, tempDir); + try (PaimonCatalogOperations ops = new PaimonCatalogOperations()) { + ops.initialize(conf, entity.toCatalogInfo(), PAIMON_PROPERTIES_METADATA); + Map map1 = Maps.newHashMap(); + map1.put(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND, "test"); + PropertiesMetadata metadata = PAIMON_PROPERTIES_METADATA.catalogPropertiesMetadata(); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> PropertiesMetadataHelpers.validatePropertyForCreate(metadata, map1)); + + Map map2 = Maps.newHashMap(); + map2.put(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND, "filesystem"); + map2.put(PaimonCatalogPropertiesMetadata.WAREHOUSE, "test"); + map2.put(PaimonCatalogPropertiesMetadata.URI, "127.0.0.1"); + Assertions.assertDoesNotThrow( + () -> PropertiesMetadataHelpers.validatePropertyForCreate(metadata, map2)); + + Map map3 = Maps.newHashMap(); + Throwable throwable = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> PropertiesMetadataHelpers.validatePropertyForCreate(metadata, map3)); + + Assertions.assertTrue( + throwable + .getMessage() + .contains(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND)); + } + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonConfig.java b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonConfig.java new file mode 100644 index 00000000000..6fd7de646dc --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonConfig.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import com.datastrato.gravitino.config.ConfigBuilder; +import com.datastrato.gravitino.config.ConfigEntry; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** Tests for {@link PaimonConfig}. */ +public class TestPaimonConfig { + + @Test + public void testLoadPaimonConfig() { + ConfigEntry testConf = new ConfigBuilder("k1").stringConf().create(); + Map properties = ImmutableMap.of(testConf.getKey(), "v1"); + + PaimonConfig paimonConfig = new PaimonConfig(); + paimonConfig.loadFromMap(properties, k -> k.startsWith("gravitino.")); + Assertions.assertNull(paimonConfig.get(testConf)); + + paimonConfig = new PaimonConfig(properties); + Assertions.assertEquals("v1", paimonConfig.get(testConf)); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonSchema.java b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonSchema.java new file mode 100644 index 00000000000..f492c30ab5c --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonSchema.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.Schema; +import com.datastrato.gravitino.SchemaChange; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.CatalogEntity; +import com.google.common.collect.Maps; +import java.io.File; +import java.time.Instant; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestPaimonSchema { + + private static final String META_LAKE_NAME = "metalake"; + + private static final String COMMENT_VALUE = "comment"; + + private static AuditInfo AUDIT_INFO = + AuditInfo.builder().withCreator("testPaimonUser").withCreateTime(Instant.now()).build(); + + private void cleanUp(PaimonCatalogOperations paimonCatalogOperations, NameIdentifier ident) { + Assertions.assertTrue(paimonCatalogOperations.dropSchema(ident, false)); + Assertions.assertThrowsExactly( + NoSuchSchemaException.class, () -> paimonCatalogOperations.loadSchema(ident)); + } + + @Test + public void testCreatePaimonSchema() { + PaimonCatalog paimonCatalog = initPaimonCatalog("testCreatePaimonSchema"); + PaimonCatalogOperations paimonCatalogOperations = (PaimonCatalogOperations) paimonCatalog.ops(); + + NameIdentifier ident = + NameIdentifier.of("metalake", paimonCatalog.name(), "test_paimon_schema"); + Map properties = Maps.newHashMap(); + properties.put("key1", "val1"); + properties.put("key2", "val2"); + + paimonCatalogOperations.dropSchema(ident, false); + + Schema schema = paimonCatalogOperations.createSchema(ident, COMMENT_VALUE, properties); + Assertions.assertEquals(ident.name(), schema.name()); + Assertions.assertEquals(COMMENT_VALUE, schema.comment()); + Assertions.assertEquals(properties, schema.properties()); + + Assertions.assertTrue(paimonCatalogOperations.schemaExists(ident)); + + Set names = + Arrays.stream(paimonCatalogOperations.listSchemas(ident.namespace())) + .map(NameIdentifier::name) + .collect(Collectors.toSet()); + Assertions.assertTrue(names.contains(ident.name())); + + // Test schema already exists + Throwable exception = + Assertions.assertThrows( + SchemaAlreadyExistsException.class, + () -> { + paimonCatalogOperations.createSchema(ident, COMMENT_VALUE, properties); + }); + Assertions.assertTrue(exception.getMessage().contains("already exists")); + + // clean up + cleanUp(paimonCatalogOperations, ident); + } + + @Test + public void testListSchema() { + PaimonCatalog paimonCatalog = initPaimonCatalog("testListPaimonSchema"); + PaimonCatalogOperations paimonCatalogOperations = (PaimonCatalogOperations) paimonCatalog.ops(); + NameIdentifier ident = + NameIdentifier.of("metalake", paimonCatalog.name(), "test_paimon_schema"); + + paimonCatalogOperations.dropSchema(ident, false); + paimonCatalogOperations.createSchema(ident, COMMENT_VALUE, Maps.newHashMap()); + + NameIdentifier[] schemas = paimonCatalogOperations.listSchemas(ident.namespace()); + Assertions.assertEquals(1, schemas.length); + Assertions.assertEquals(ident.name(), schemas[0].name()); + Assertions.assertEquals(ident.namespace(), schemas[0].namespace()); + + // clean up + cleanUp(paimonCatalogOperations, ident); + } + + @Test + public void testAlterSchema() { + PaimonCatalog paimonCatalog = initPaimonCatalog("testListPaimonSchema"); + PaimonCatalogOperations paimonCatalogOperations = (PaimonCatalogOperations) paimonCatalog.ops(); + + NameIdentifier ident = + NameIdentifier.of("metalake", paimonCatalog.name(), "test_paimon_schema"); + Map properties = Maps.newHashMap(); + properties.put("key1", "val1"); + properties.put("key2", "val2"); + + paimonCatalogOperations.dropSchema(ident, false); + PaimonSchema paimonSchema = + paimonCatalogOperations.createSchema(ident, COMMENT_VALUE, properties); + Assertions.assertTrue(paimonCatalogOperations.schemaExists(ident)); + properties.forEach( + (k, v) -> Assertions.assertEquals(v, paimonSchema.toPaimonProperties().get(k))); + + // schema properties of FilesystemCatalog is empty when loadDatabase. + Map properties1 = paimonCatalogOperations.loadSchema(ident).properties(); + Assertions.assertEquals(0, properties1.size()); + + Assertions.assertThrowsExactly( + UnsupportedOperationException.class, + () -> paimonCatalogOperations.alterSchema(ident, SchemaChange.removeProperty("key1"))); + + // clean up + cleanUp(paimonCatalogOperations, ident); + } + + @Test + public void testDropSchema() { + PaimonCatalog paimonCatalog = initPaimonCatalog("testListPaimonSchema"); + PaimonCatalogOperations paimonCatalogOperations = (PaimonCatalogOperations) paimonCatalog.ops(); + + NameIdentifier ident = + NameIdentifier.of("metalake", paimonCatalog.name(), "test_paimon_schema"); + Map properties = Maps.newHashMap(); + properties.put("key1", "val1"); + properties.put("key2", "val2"); + + paimonCatalogOperations.dropSchema(ident, false); + paimonCatalogOperations.createSchema(ident, COMMENT_VALUE, properties); + Assertions.assertTrue(paimonCatalogOperations.schemaExists(ident)); + Assertions.assertTrue(paimonCatalogOperations.dropSchema(ident, false)); + Assertions.assertFalse(paimonCatalogOperations.schemaExists(ident)); + + Assertions.assertFalse(paimonCatalogOperations.dropSchema(ident, false)); + } + + private PaimonCatalog initPaimonCatalog(String name) { + CatalogEntity entity = + CatalogEntity.builder() + .withId(1L) + .withName(name) + .withNamespace(Namespace.of(META_LAKE_NAME)) + .withType(PaimonCatalog.Type.RELATIONAL) + .withProvider("lakehouse-paimon") + .withAuditInfo(AUDIT_INFO) + .build(); + + Map conf = Maps.newHashMap(); + conf.put(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND, "filesystem"); + conf.put( + PaimonCatalogPropertiesMetadata.WAREHOUSE, + String.join( + File.separator, System.getProperty("java.io.tmpdir"), "paimon_catalog_warehouse")); + return new PaimonCatalog().withCatalogConf(conf).withCatalogEntity(entity); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java new file mode 100644 index 00000000000..5c45dcefeff --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java @@ -0,0 +1,584 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon.integration.test; + +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.Schema; +import com.datastrato.gravitino.SchemaChange; +import com.datastrato.gravitino.SupportsSchemas; +import com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonCatalogPropertiesMetadata; +import com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonConfig; +import com.datastrato.gravitino.catalog.lakehouse.paimon.utils.CatalogUtils; +import com.datastrato.gravitino.client.GravitinoMetalake; +import com.datastrato.gravitino.dto.util.DTOConverters; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; +import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.util.AbstractIT; +import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.TableCatalog; +import com.datastrato.gravitino.rel.expressions.distributions.Distribution; +import com.datastrato.gravitino.rel.expressions.distributions.Distributions; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; +import com.datastrato.gravitino.rel.expressions.sorts.SortOrders; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.expressions.transforms.Transforms; +import com.datastrato.gravitino.rel.types.Types; +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import org.apache.paimon.catalog.Catalog.DatabaseNotExistException; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.paimon.types.TimestampType; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.platform.commons.util.StringUtils; + +public abstract class CatalogPaimonBaseIT extends AbstractIT { + + protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + protected String WAREHOUSE; + protected String TYPE; + + private static final String provider = "lakehouse-paimon"; + private static final String catalog_comment = "catalog_comment"; + private static final String schema_comment = "schema_comment"; + private static final String table_comment = "table_comment"; + private static final String PAIMON_COL_NAME1 = "paimon_col_name1"; + private static final String PAIMON_COL_NAME2 = "paimon_col_name2"; + private static final String PAIMON_COL_NAME3 = "paimon_col_name3"; + private static final String PAIMON_COL_NAME4 = "paimon_col_name4"; + private String metalakeName = GravitinoITUtils.genRandomName("paimon_it_metalake"); + private String catalogName = GravitinoITUtils.genRandomName("paimon_it_catalog"); + private String schemaName = GravitinoITUtils.genRandomName("paimon_it_schema"); + private String tableName = GravitinoITUtils.genRandomName("paimon_it_table"); + private static String INSERT_BATCH_WITHOUT_PARTITION_TEMPLATE = "INSERT INTO paimon.%s VALUES %s"; + private static final String SELECT_ALL_TEMPLATE = "SELECT * FROM paimon.%s"; + private GravitinoMetalake metalake; + private Catalog catalog; + private org.apache.paimon.catalog.Catalog paimonCatalog; + private SparkSession spark; + private Map catalogProperties; + + @BeforeAll + public void startup() { + containerSuite.startHiveContainer(); + catalogProperties = initPaimonCatalogProperties(); + createMetalake(); + createCatalog(); + createSchema(); + initSparkEnv(); + } + + @AfterAll + public void stop() { + clearTableAndSchema(); + metalake.dropCatalog(catalogName); + client.dropMetalake(metalakeName); + if (spark != null) { + spark.close(); + } + } + + @AfterEach + private void resetSchema() { + clearTableAndSchema(); + createSchema(); + } + + protected abstract Map initPaimonCatalogProperties(); + + @Test + void testPaimonSchemaOperations() throws DatabaseNotExistException { + SupportsSchemas schemas = catalog.asSchemas(); + + // create schema check. + String testSchemaName = GravitinoITUtils.genRandomName("test_schema_1"); + NameIdentifier schemaIdent = NameIdentifier.of(metalakeName, catalogName, testSchemaName); + Map schemaProperties = Maps.newHashMap(); + schemaProperties.put("key1", "val1"); + schemaProperties.put("key2", "val2"); + schemas.createSchema(schemaIdent.name(), schema_comment, schemaProperties); + + Set schemaNames = new HashSet<>(Arrays.asList(schemas.listSchemas())); + Assertions.assertTrue(schemaNames.contains(testSchemaName)); + List paimonDatabaseNames = paimonCatalog.listDatabases(); + Assertions.assertTrue(paimonDatabaseNames.contains(testSchemaName)); + + // load schema check. + Schema schema = schemas.loadSchema(schemaIdent.name()); + // database properties is empty for Paimon FilesystemCatalog. + Assertions.assertTrue(schema.properties().isEmpty()); + Assertions.assertTrue(paimonCatalog.loadDatabaseProperties(schemaIdent.name()).isEmpty()); + + Map emptyMap = Collections.emptyMap(); + Assertions.assertThrows( + SchemaAlreadyExistsException.class, + () -> schemas.createSchema(schemaIdent.name(), schema_comment, emptyMap)); + + // alter schema check. + // alter schema operation is unsupported. + Assertions.assertThrowsExactly( + UnsupportedOperationException.class, + () -> schemas.alterSchema(schemaIdent.name(), SchemaChange.setProperty("k1", "v1"))); + + // drop schema check. + schemas.dropSchema(schemaIdent.name(), false); + Assertions.assertThrows( + NoSuchSchemaException.class, () -> schemas.loadSchema(schemaIdent.name())); + Assertions.assertThrows( + DatabaseNotExistException.class, + () -> { + paimonCatalog.loadDatabaseProperties(schemaIdent.name()); + }); + + schemaNames = new HashSet<>(Arrays.asList(schemas.listSchemas())); + Assertions.assertFalse(schemaNames.contains(testSchemaName)); + Assertions.assertFalse(schemas.dropSchema(schemaIdent.name(), false)); + Assertions.assertFalse(schemas.dropSchema("no-exits", false)); + + // list schema check. + schemaNames = new HashSet<>(Arrays.asList(schemas.listSchemas())); + Assertions.assertFalse(schemaNames.contains(testSchemaName)); + paimonDatabaseNames = paimonCatalog.listDatabases(); + Assertions.assertFalse(paimonDatabaseNames.contains(testSchemaName)); + } + + @Test + void testCreateTableWithNullComment() { + Column[] columns = createColumns(); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); + + TableCatalog tableCatalog = catalog.asTableCatalog(); + Table createdTable = + tableCatalog.createTable(tableIdentifier, columns, null, null, null, null, null); + Assertions.assertNull(createdTable.comment()); + + Table loadTable = tableCatalog.loadTable(tableIdentifier); + Assertions.assertNull(loadTable.comment()); + } + + @Test + void testCreateAndLoadPaimonTable() + throws org.apache.paimon.catalog.Catalog.TableNotExistException { + // Create table from Gravitino API + Column[] columns = createColumns(); + + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); + Distribution distribution = Distributions.NONE; + + Transform[] partitioning = Transforms.EMPTY_TRANSFORM; + SortOrder[] sortOrders = SortOrders.NONE; + Map properties = createProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + Table createdTable = + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + partitioning, + distribution, + sortOrders); + Assertions.assertEquals(createdTable.name(), tableName); + Map resultProp = createdTable.properties(); + for (Map.Entry entry : properties.entrySet()) { + Assertions.assertTrue(resultProp.containsKey(entry.getKey())); + Assertions.assertEquals(entry.getValue(), resultProp.get(entry.getKey())); + } + Assertions.assertEquals(createdTable.columns().length, columns.length); + + for (int i = 0; i < columns.length; i++) { + Assertions.assertEquals(DTOConverters.toDTO(columns[i]), createdTable.columns()[i]); + } + + Table loadTable = tableCatalog.loadTable(tableIdentifier); + Assertions.assertEquals(tableName, loadTable.name()); + Assertions.assertEquals(table_comment, loadTable.comment()); + resultProp = loadTable.properties(); + for (Map.Entry entry : properties.entrySet()) { + Assertions.assertTrue(resultProp.containsKey(entry.getKey())); + Assertions.assertEquals(entry.getValue(), resultProp.get(entry.getKey())); + } + Assertions.assertEquals(loadTable.columns().length, columns.length); + for (int i = 0; i < columns.length; i++) { + Assertions.assertEquals(DTOConverters.toDTO(columns[i]), loadTable.columns()[i]); + } + + // catalog load check + org.apache.paimon.table.Table table = + paimonCatalog.getTable(Identifier.create(schemaName, tableName)); + Assertions.assertEquals(tableName, table.name()); + Assertions.assertTrue(table.comment().isPresent()); + Assertions.assertEquals(table_comment, table.comment().get()); + resultProp = table.options(); + for (Map.Entry entry : properties.entrySet()) { + Assertions.assertTrue(resultProp.containsKey(entry.getKey())); + Assertions.assertEquals(entry.getValue(), resultProp.get(entry.getKey())); + } + + Assertions.assertInstanceOf(FileStoreTable.class, table); + FileStoreTable fileStoreTable = (FileStoreTable) table; + + TableSchema schema = fileStoreTable.schema(); + Assertions.assertEquals(schema.fields().size(), columns.length); + for (int i = 0; i < columns.length; i++) { + Assertions.assertEquals(columns[i].name(), schema.fieldNames().get(i)); + } + Assertions.assertEquals(partitioning.length, fileStoreTable.partitionKeys().size()); + + Assertions.assertThrows( + TableAlreadyExistsException.class, + () -> + catalog + .asTableCatalog() + .createTable( + tableIdentifier, + columns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + distribution, + sortOrders)); + } + + @Test + void testCreateTableWithTimestampColumn() + throws org.apache.paimon.catalog.Catalog.TableNotExistException { + Column col1 = Column.of("paimon_column_1", Types.TimestampType.withTimeZone(), "col_1_comment"); + Column col2 = + Column.of("paimon_column_2", Types.TimestampType.withoutTimeZone(), "col_2_comment"); + + Column[] columns = new Column[] {col1, col2}; + + String timestampTableName = "timestamp_table"; + + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, timestampTableName); + + Map properties = createProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + Table createdTable = + tableCatalog.createTable(tableIdentifier, columns, table_comment, properties); + Assertions.assertEquals("paimon_column_1", createdTable.columns()[0].name()); + Assertions.assertEquals( + Types.TimestampType.withTimeZone(), createdTable.columns()[0].dataType()); + Assertions.assertEquals("col_1_comment", createdTable.columns()[0].comment()); + Assertions.assertTrue(createdTable.columns()[0].nullable()); + + Assertions.assertEquals("paimon_column_2", createdTable.columns()[1].name()); + Assertions.assertEquals( + Types.TimestampType.withoutTimeZone(), createdTable.columns()[1].dataType()); + Assertions.assertEquals("col_2_comment", createdTable.columns()[1].comment()); + Assertions.assertTrue(createdTable.columns()[1].nullable()); + + Table loadTable = tableCatalog.loadTable(tableIdentifier); + Assertions.assertEquals("paimon_column_1", loadTable.columns()[0].name()); + Assertions.assertEquals(Types.TimestampType.withTimeZone(), loadTable.columns()[0].dataType()); + Assertions.assertEquals("col_1_comment", loadTable.columns()[0].comment()); + Assertions.assertTrue(loadTable.columns()[0].nullable()); + + Assertions.assertEquals("paimon_column_2", loadTable.columns()[1].name()); + Assertions.assertEquals( + Types.TimestampType.withoutTimeZone(), loadTable.columns()[1].dataType()); + Assertions.assertEquals("col_2_comment", loadTable.columns()[1].comment()); + Assertions.assertTrue(loadTable.columns()[1].nullable()); + + org.apache.paimon.table.Table table = + paimonCatalog.getTable(Identifier.create(schemaName, timestampTableName)); + Assertions.assertInstanceOf(FileStoreTable.class, table); + FileStoreTable fileStoreTable = (FileStoreTable) table; + TableSchema tableSchema = fileStoreTable.schema(); + Assertions.assertEquals("paimon_column_1", tableSchema.fields().get(0).name()); + Assertions.assertEquals( + new LocalZonedTimestampType().nullable(), tableSchema.fields().get(0).type()); + Assertions.assertEquals("col_1_comment", tableSchema.fields().get(0).description()); + + Assertions.assertEquals("paimon_column_2", tableSchema.fields().get(1).name()); + Assertions.assertEquals(new TimestampType().nullable(), tableSchema.fields().get(1).type()); + Assertions.assertEquals("col_2_comment", tableSchema.fields().get(1).description()); + } + + @Test + void testListAndDropPaimonTable() throws DatabaseNotExistException { + Column[] columns = createColumns(); + + String tableName1 = "table_1"; + + NameIdentifier table1 = NameIdentifier.of(schemaName, tableName1); + + Map properties = createProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + tableCatalog.createTable( + table1, + columns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0]); + NameIdentifier[] nameIdentifiers = tableCatalog.listTables(Namespace.of(schemaName)); + Assertions.assertEquals(1, nameIdentifiers.length); + Assertions.assertEquals("table_1", nameIdentifiers[0].name()); + + List tableIdentifiers = paimonCatalog.listTables(schemaName); + Assertions.assertEquals(1, tableIdentifiers.size()); + Assertions.assertEquals("table_1", tableIdentifiers.get(0)); + + String tableName2 = "table_2"; + + NameIdentifier table2 = NameIdentifier.of(schemaName, tableName2); + tableCatalog.createTable( + table2, + columns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0]); + nameIdentifiers = tableCatalog.listTables(Namespace.of(schemaName)); + Assertions.assertEquals(2, nameIdentifiers.length); + Assertions.assertEquals("table_1", nameIdentifiers[0].name()); + Assertions.assertEquals("table_2", nameIdentifiers[1].name()); + + tableIdentifiers = paimonCatalog.listTables(schemaName); + Assertions.assertEquals(2, tableIdentifiers.size()); + Assertions.assertEquals("table_1", tableIdentifiers.get(0)); + Assertions.assertEquals("table_2", tableIdentifiers.get(1)); + + Assertions.assertDoesNotThrow(() -> tableCatalog.dropTable(table1)); + + nameIdentifiers = tableCatalog.listTables(Namespace.of(schemaName)); + Assertions.assertEquals(1, nameIdentifiers.length); + Assertions.assertEquals("table_2", nameIdentifiers[0].name()); + + Assertions.assertDoesNotThrow(() -> tableCatalog.dropTable(table2)); + Namespace schemaNamespace = Namespace.of(schemaName); + nameIdentifiers = tableCatalog.listTables(schemaNamespace); + Assertions.assertEquals(0, nameIdentifiers.length); + + Assertions.assertEquals(0, paimonCatalog.listTables(schemaName).size()); + } + + @Test + void testOperationDataOfPaimonTable() { + Column[] columns = createColumns(); + String testTableName = GravitinoITUtils.genRandomName("test_table"); + SortOrder[] sortOrders = SortOrders.NONE; + Transform[] transforms = Transforms.EMPTY_TRANSFORM; + catalog + .asTableCatalog() + .createTable( + NameIdentifier.of(schemaName, testTableName), + columns, + table_comment, + createProperties(), + transforms, + Distributions.NONE, + sortOrders); + List values = getValues(); + String dbTable = String.join(".", schemaName, testTableName); + // insert data + String insertSQL = + String.format(INSERT_BATCH_WITHOUT_PARTITION_TEMPLATE, dbTable, String.join(", ", values)); + spark.sql(insertSQL); + + // select data + Dataset sql = spark.sql(String.format(SELECT_ALL_TEMPLATE, dbTable)); + Assertions.assertEquals(4, sql.count()); + Row[] result = (Row[]) sql.sort(PAIMON_COL_NAME1).collect(); + LocalDate currentDate = LocalDate.now(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + for (int i = 0; i < result.length; i++) { + LocalDate previousDay = currentDate.minusDays(i + 1); + Assertions.assertEquals( + String.format( + "[%s,%s,data%s,[%s,string%s,[%s,inner%s]]]", + i + 1, previousDay.format(formatter), i + 1, (i + 1) * 10, i + 1, i + 1, i + 1), + result[i].toString()); + } + + // update data + spark.sql( + String.format( + "UPDATE paimon.%s SET %s = 100 WHERE %s = 1", + dbTable, PAIMON_COL_NAME1, PAIMON_COL_NAME1)); + sql = spark.sql(String.format(SELECT_ALL_TEMPLATE, dbTable)); + Assertions.assertEquals(4, sql.count()); + result = (Row[]) sql.sort(PAIMON_COL_NAME1).collect(); + for (int i = 0; i < result.length; i++) { + if (i == result.length - 1) { + LocalDate previousDay = currentDate.minusDays(1); + Assertions.assertEquals( + String.format( + "[100,%s,data%s,[%s,string%s,[%s,inner%s]]]", + previousDay.format(formatter), 1, 10, 1, 1, 1), + result[i].toString()); + } else { + LocalDate previousDay = currentDate.minusDays(i + 2); + Assertions.assertEquals( + String.format( + "[%s,%s,data%s,[%s,string%s,[%s,inner%s]]]", + i + 2, previousDay.format(formatter), i + 2, (i + 2) * 10, i + 2, i + 2, i + 2), + result[i].toString()); + } + } + // delete data + spark.sql(String.format("DELETE FROM paimon.%s WHERE %s = 100", dbTable, PAIMON_COL_NAME1)); + sql = spark.sql(String.format(SELECT_ALL_TEMPLATE, dbTable)); + Assertions.assertEquals(3, sql.count()); + result = (Row[]) sql.sort(PAIMON_COL_NAME1).collect(); + for (int i = 0; i < result.length; i++) { + LocalDate previousDay = currentDate.minusDays(i + 2); + Assertions.assertEquals( + String.format( + "[%s,%s,data%s,[%s,string%s,[%s,inner%s]]]", + i + 2, previousDay.format(formatter), i + 2, (i + 2) * 10, i + 2, i + 2, i + 2), + result[i].toString()); + } + } + + private static @NotNull List getValues() { + List values = new ArrayList<>(); + for (int i = 1; i < 5; i++) { + String structValue = + String.format( + "STRUCT(%d, 'string%d', %s)", + i * 10, // integer_field + i, // string_field + String.format( + "STRUCT(%d, 'inner%d')", + i, i) // struct_field, alternating NULL and non-NULL values + ); + values.add( + String.format("(%d, date_sub(current_date(), %d), 'data%d', %s)", i, i, i, structValue)); + } + return values; + } + + private void clearTableAndSchema() { + if (catalog.asSchemas().schemaExists(schemaName)) { + catalog.asSchemas().dropSchema(schemaName, true); + } + } + + private void createMetalake() { + GravitinoMetalake createdMetalake = + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); + Assertions.assertEquals(createdMetalake, loadMetalake); + + metalake = loadMetalake; + } + + private void createCatalog() { + Catalog createdCatalog = + metalake.createCatalog( + catalogName, Catalog.Type.RELATIONAL, provider, catalog_comment, catalogProperties); + Catalog loadCatalog = metalake.loadCatalog(catalogName); + Assertions.assertEquals(createdCatalog, loadCatalog); + catalog = loadCatalog; + + String type = + catalogProperties + .get(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND) + .toLowerCase(Locale.ROOT); + Preconditions.checkArgument( + StringUtils.isNotBlank(type), "Paimon Catalog backend type can not be null or empty."); + catalogProperties.put(PaimonCatalogPropertiesMetadata.PAIMON_METASTORE, type); + paimonCatalog = CatalogUtils.loadCatalogBackend(new PaimonConfig(catalogProperties)); + } + + private void createSchema() { + NameIdentifier ident = NameIdentifier.of(metalakeName, catalogName, schemaName); + Map prop = Maps.newHashMap(); + prop.put("key1", "val1"); + prop.put("key2", "val2"); + + Schema createdSchema = catalog.asSchemas().createSchema(ident.name(), schema_comment, prop); + // database properties is empty for Paimon FilesystemCatalog. + Schema loadSchema = catalog.asSchemas().loadSchema(ident.name()); + Assertions.assertEquals(createdSchema.name(), loadSchema.name()); + Assertions.assertTrue(loadSchema.properties().isEmpty()); + } + + private Column[] createColumns() { + Column col1 = Column.of(PAIMON_COL_NAME1, Types.IntegerType.get(), "col_1_comment"); + Column col2 = Column.of(PAIMON_COL_NAME2, Types.DateType.get(), "col_2_comment"); + Column col3 = Column.of(PAIMON_COL_NAME3, Types.StringType.get(), "col_3_comment"); + Types.StructType structTypeInside = + Types.StructType.of( + Types.StructType.Field.notNullField("integer_field_inside", Types.IntegerType.get()), + Types.StructType.Field.notNullField( + "string_field_inside", Types.StringType.get(), "string field inside")); + Types.StructType structType = + Types.StructType.of( + Types.StructType.Field.notNullField("integer_field", Types.IntegerType.get()), + Types.StructType.Field.notNullField( + "string_field", Types.StringType.get(), "string field"), + Types.StructType.Field.nullableField("struct_field", structTypeInside, "struct field")); + Column col4 = Column.of(PAIMON_COL_NAME4, structType, "col_4_comment"); + return new Column[] {col1, col2, col3, col4}; + } + + private Map createProperties() { + Map properties = Maps.newHashMap(); + properties.put("key1", "val1"); + properties.put("key2", "val2"); + return properties; + } + + private void initSparkEnv() { + spark = + SparkSession.builder() + .master("local[1]") + .appName("Paimon Catalog integration test") + .config("spark.sql.warehouse.dir", WAREHOUSE) + .config("spark.sql.catalog.paimon", "org.apache.paimon.spark.SparkCatalog") + .config("spark.sql.catalog.paimon.warehouse", WAREHOUSE) + .config( + "spark.sql.extensions", + "org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions") + .enableHiveSupport() + .getOrCreate(); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonFileSystemIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonFileSystemIT.java new file mode 100644 index 00000000000..cd14c7c9acb --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonFileSystemIT.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon.integration.test; + +import com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonCatalogPropertiesMetadata; +import com.datastrato.gravitino.integration.test.container.HiveContainer; +import com.google.common.collect.Maps; +import java.util.Map; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestInstance; + +@Tag("gravitino-docker-test") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class CatalogPaimonFileSystemIT extends CatalogPaimonBaseIT { + + @Override + protected Map initPaimonCatalogProperties() { + + Map catalogProperties = Maps.newHashMap(); + catalogProperties.put("key1", "val1"); + catalogProperties.put("key2", "val2"); + + TYPE = "filesystem"; + WAREHOUSE = + String.format( + "hdfs://%s:%d/user/hive/warehouse-catalog-paimon/", + containerSuite.getHiveContainer().getContainerIpAddress(), + HiveContainer.HDFS_DEFAULTFS_PORT); + + catalogProperties.put(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND, TYPE); + catalogProperties.put(PaimonCatalogPropertiesMetadata.WAREHOUSE, WAREHOUSE); + + return catalogProperties; + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/ops/TestPaimonCatalogOps.java b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/ops/TestPaimonCatalogOps.java new file mode 100644 index 00000000000..784eec10c21 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/ops/TestPaimonCatalogOps.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon.ops; + +import static org.apache.paimon.CoreOptions.BUCKET; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonCatalogPropertiesMetadata; +import com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonConfig; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import java.io.File; +import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.DateType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.TimestampType; +import org.apache.paimon.types.VarCharType; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** Tests for {@link PaimonCatalogOps}. */ +public class TestPaimonCatalogOps { + + private PaimonCatalogOps paimonCatalogOps; + @TempDir private File warehouse; + + private static final String DATABASE = "test_table_ops_database"; + private static final String TABLE = "test_table_ops_table"; + private static final String COMMENT = "table_ops_table_comment"; + private static final NameIdentifier IDENTIFIER = NameIdentifier.of(Namespace.of(DATABASE), TABLE); + private static final Map OPTIONS = ImmutableMap.of(BUCKET.key(), "10"); + + @BeforeEach + public void setUp() throws Exception { + paimonCatalogOps = + new PaimonCatalogOps( + new PaimonConfig( + ImmutableMap.of(PaimonCatalogPropertiesMetadata.WAREHOUSE, warehouse.getPath()))); + createDatabase(); + } + + @AfterEach + public void tearDown() throws Exception { + dropDatabase(); + if (paimonCatalogOps != null) { + paimonCatalogOps.close(); + } + } + + @Test + void testTableOperations() throws Exception { + // list tables + Assertions.assertEquals( + 0, paimonCatalogOps.listTables(IDENTIFIER.namespace().toString()).size()); + + // create table + Pair tableInfo = + Pair.of( + IDENTIFIER.toString(), + Schema.newBuilder() + .column("col_1", DataTypes.INT().notNull(), IntType.class.getSimpleName()) + .column("col_2", DataTypes.STRING(), VarCharType.class.getSimpleName()) + .column("col_3", DataTypes.STRING().notNull(), VarCharType.class.getSimpleName()) + .column( + "col_4", + DataTypes.ARRAY( + RowType.builder() + .field( + "sub_col_1", + DataTypes.DATE(), + RowType.class.getSimpleName() + DateType.class.getSimpleName()) + .field( + "sub_col_2", + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + RowType.class.getSimpleName() + MapType.class.getSimpleName()) + .field( + "sub_col_3", + DataTypes.TIMESTAMP().notNull(), + RowType.class.getSimpleName() + TimestampType.class.getSimpleName()) + .build()), + ArrayType.class.getSimpleName()) + .comment(COMMENT) + .options(OPTIONS) + .build()); + paimonCatalogOps.createTable(tableInfo.getKey(), tableInfo.getValue()); + + // load table + Table table = paimonCatalogOps.loadTable(IDENTIFIER.toString()); + + assertEquals(TABLE, table.name()); + assertTrue(table.comment().isPresent()); + assertEquals( + RowType.builder() + .field("col_1", DataTypes.INT().notNull(), IntType.class.getSimpleName()) + .field("col_2", DataTypes.STRING(), VarCharType.class.getSimpleName()) + .field("col_3", DataTypes.STRING().notNull(), VarCharType.class.getSimpleName()) + .field( + "col_4", + DataTypes.ARRAY( + RowType.builder() + .field( + "sub_col_1", + DataTypes.DATE(), + RowType.class.getSimpleName() + DateType.class.getSimpleName()) + .field( + "sub_col_2", + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + RowType.class.getSimpleName() + MapType.class.getSimpleName()) + .field( + "sub_col_3", + DataTypes.TIMESTAMP().notNull(), + RowType.class.getSimpleName() + TimestampType.class.getSimpleName()) + .build()), + ArrayType.class.getSimpleName()) + .build() + .toString(), + table.rowType().toString()); + assertEquals(COMMENT, table.comment().get()); + assertEquals(OPTIONS.get(BUCKET.key()), table.options().get(BUCKET.key())); + + // TODO: alter table is unsupported now. + + // drop table + Assertions.assertDoesNotThrow(() -> paimonCatalogOps.dropTable(IDENTIFIER.toString())); + Assertions.assertThrowsExactly( + Catalog.TableNotExistException.class, + () -> paimonCatalogOps.dropTable(IDENTIFIER.toString())); + + // list table again + Assertions.assertEquals( + 0, paimonCatalogOps.listTables(IDENTIFIER.namespace().toString()).size()); + + // create a new table to make database not empty to test drop database cascade + paimonCatalogOps.createTable(tableInfo.getKey(), tableInfo.getValue()); + Assertions.assertNotNull(paimonCatalogOps.loadTable(IDENTIFIER.toString())); + } + + private void createDatabase() throws Exception { + // list databases + assertEquals(0, paimonCatalogOps.listDatabases().size()); + + // create database + paimonCatalogOps.createDatabase(DATABASE, Maps.newHashMap()); + assertEquals(1, paimonCatalogOps.listDatabases().size()); + // load database + assertNotNull(paimonCatalogOps.loadDatabase(DATABASE)); + } + + private void dropDatabase() throws Exception { + Assertions.assertEquals(1, paimonCatalogOps.listDatabases().size()); + Assertions.assertEquals(1, paimonCatalogOps.listTables(DATABASE).size()); + paimonCatalogOps.dropDatabase(DATABASE, true); + Assertions.assertTrue(paimonCatalogOps.listDatabases().isEmpty()); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TestCatalogUtils.java b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TestCatalogUtils.java new file mode 100644 index 00000000000..9186ba27e7d --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TestCatalogUtils.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.datastrato.gravitino.catalog.lakehouse.paimon.utils; + +import static com.datastrato.gravitino.catalog.lakehouse.paimon.utils.CatalogUtils.loadCatalogBackend; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; + +import com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonCatalogBackend; +import com.datastrato.gravitino.catalog.lakehouse.paimon.PaimonConfig; +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.util.Locale; +import java.util.function.Consumer; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.FileSystemCatalog; +import org.apache.paimon.factories.FactoryException; +import org.junit.jupiter.api.Test; + +/** Tests for {@link CatalogUtils}. */ +public class TestCatalogUtils { + + @Test + void testLoadCatalogBackend() throws Exception { + // Test load FileSystemCatalog for filesystem metastore. + assertCatalog(PaimonCatalogBackend.FILESYSTEM.name(), FileSystemCatalog.class); + // Test load catalog exception for other metastore. + assertThrowsExactly(FactoryException.class, () -> assertCatalog("other", catalog -> {})); + } + + private void assertCatalog(String metastore, Class expected) throws Exception { + assertCatalog( + metastore.toLowerCase(Locale.ROOT), catalog -> assertEquals(expected, catalog.getClass())); + } + + private void assertCatalog(String metastore, Consumer consumer) throws Exception { + try (Catalog catalog = + loadCatalogBackend( + new PaimonConfig( + ImmutableMap.of( + PaimonConfig.CATALOG_BACKEND.getKey(), + metastore, + PaimonConfig.CATALOG_WAREHOUSE.getKey(), + String.join( + File.separator, + System.getProperty("java.io.tmpdir"), + "paimon_catalog_warehouse"), + PaimonConfig.CATALOG_URI.getKey(), + "uri")))) { + consumer.accept(catalog); + } + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TestTypeUtils.java b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TestTypeUtils.java new file mode 100644 index 00000000000..fe47d9bbcae --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/utils/TestTypeUtils.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.catalog.lakehouse.paimon.utils; + +import static com.datastrato.gravitino.catalog.lakehouse.paimon.utils.TypeUtils.fromPaimonType; +import static com.datastrato.gravitino.catalog.lakehouse.paimon.utils.TypeUtils.toPaimonType; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Type.Name; +import com.datastrato.gravitino.rel.types.Types; +import java.util.Arrays; +import java.util.function.Consumer; +import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.CharType; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarBinaryType; +import org.apache.paimon.types.VarCharType; +import org.junit.jupiter.api.Test; + +/** Tests for {@link TypeUtils}. */ +public class TestTypeUtils { + + @Test + void testFromAndToPaimonType() { + // Test supported data types. + RowType rowType = + RowType.builder() + .fields( + DataTypes.VARCHAR(10), + DataTypes.STRING(), + DataTypes.BOOLEAN(), + DataTypes.BINARY(BinaryType.MAX_LENGTH), + DataTypes.DECIMAL(8, 3), + DataTypes.TINYINT(), + DataTypes.SMALLINT(), + DataTypes.INT(), + DataTypes.BIGINT(), + DataTypes.FLOAT(), + DataTypes.DOUBLE(), + DataTypes.DATE(), + DataTypes.TIME(), + DataTypes.TIMESTAMP(), + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(), + DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()), + DataTypes.ARRAY(DataTypes.INT())) + .build(); + + Type gravitinoDataType = toGravitinoDataType(rowType); + + DataType paimonDataType = toPaimonType(gravitinoDataType); + + assertEquals(rowType, paimonDataType); + } + + @Test + void testUnparsedType() { + Arrays.asList( + DataTypes.CHAR(CharType.MAX_LENGTH), DataTypes.VARBINARY(VarBinaryType.MAX_LENGTH)) + .forEach(this::toGravitinoDataType); + } + + @Test + void testUnsupportedType() { + // Test UnsupportedOperationException with IntervalYearType, IntervalDayType, FixedCharType, + // UUIDType, FixedType, UnionType, NullType for toPaimonType. + Arrays.asList( + Types.IntervalYearType.get(), + Types.IntervalDayType.get(), + Types.FixedCharType.of(10), + Types.UUIDType.get(), + Types.FixedType.of(20), + Types.UnionType.of(Types.IntegerType.get()), + Types.NullType.get(), + Types.UnparsedType.of("unparsed")) + .forEach(this::checkUnsupportedType); + } + + private Type toGravitinoDataType(DataType dataType) { + switch (dataType.getTypeRoot()) { + case VARCHAR: + if (((VarCharType) dataType).getLength() == Integer.MAX_VALUE) { + return checkDataType(dataType, Name.STRING); + } else { + return checkDataType( + dataType, + Name.VARCHAR, + type -> + assertEquals( + ((VarCharType) dataType).getLength(), ((Types.VarCharType) type).length())); + } + case BOOLEAN: + return checkDataType(dataType, Name.BOOLEAN); + case BINARY: + return checkDataType(dataType, Name.BINARY); + case DECIMAL: + return checkDataType( + dataType, + Name.DECIMAL, + type -> { + assertEquals( + ((DecimalType) dataType).getPrecision(), ((Types.DecimalType) type).precision()); + assertEquals(((DecimalType) dataType).getScale(), ((Types.DecimalType) type).scale()); + }); + case TINYINT: + return checkDataType(dataType, Name.BYTE); + case SMALLINT: + return checkDataType(dataType, Name.SHORT); + case INTEGER: + return checkDataType(dataType, Name.INTEGER); + case BIGINT: + return checkDataType(dataType, Name.LONG); + case FLOAT: + return checkDataType(dataType, Name.FLOAT); + case DOUBLE: + return checkDataType(dataType, Name.DOUBLE); + case DATE: + return checkDataType(dataType, Name.DATE); + case TIME_WITHOUT_TIME_ZONE: + return checkDataType(dataType, Name.TIME); + case TIMESTAMP_WITHOUT_TIME_ZONE: + return checkDataType( + dataType, + Name.TIMESTAMP, + type -> assertFalse(((Types.TimestampType) type).hasTimeZone())); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return checkDataType( + dataType, + Name.TIMESTAMP, + type -> assertTrue(((Types.TimestampType) type).hasTimeZone())); + case ARRAY: + return checkDataType( + dataType, + Name.LIST, + type -> assertTrue(((Types.ListType) type).elementType() instanceof Types.IntegerType)); + case MULTISET: + return checkDataType( + dataType, + Name.MAP, + type -> { + assertEquals( + fromPaimonType(((MultisetType) dataType).getElementType()), + ((Types.MapType) type).keyType()); + assertTrue(((Types.MapType) type).valueType() instanceof Types.IntegerType); + }); + case MAP: + return checkDataType( + dataType, + Name.MAP, + type -> { + assertEquals( + fromPaimonType(((MapType) dataType).getKeyType()), + ((Types.MapType) type).keyType()); + assertEquals( + fromPaimonType(((MapType) dataType).getValueType()), + ((Types.MapType) type).valueType()); + }); + case ROW: + return checkDataType( + dataType, + Name.STRUCT, + type -> ((RowType) dataType).getFieldTypes().forEach(this::toGravitinoDataType)); + default: + return checkDataType(dataType, Name.UNPARSED); + } + } + + private Type checkDataType(DataType dataType, Name expected) { + return checkDataType(dataType, expected, type -> {}); + } + + private Type checkDataType(DataType dataType, Name expected, Consumer consumer) { + Type actual = fromPaimonType(dataType); + assertEquals(expected, actual.name()); + consumer.accept(actual); + return actual; + } + + private void checkUnsupportedType(Type type) { + UnsupportedOperationException exception = + assertThrowsExactly(UnsupportedOperationException.class, () -> toPaimonType(type)); + assertEquals( + String.format("Paimon does not support Gravitino %s data type.", type.simpleString()), + exception.getMessage()); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/test/resources/log4j2.properties b/catalogs/catalog-lakehouse-paimon/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..e9f59bdebd3 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/test/resources/log4j2.properties @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Set to debug or trace if log4j initialization is failing +status = info + +# Name of the configuration +name = ConsoleLogConfig + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Log files location +property.logPath = ${sys:gravitino.log.path:-catalog-lakehouse-paimon/build/paimon-integration-test.log} + +# File appender configuration +appender.file.type = File +appender.file.name = fileLogger +appender.file.fileName = ${logPath} +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Root logger level +rootLogger.level = info + +# Root logger referring to console and file appenders +rootLogger.appenderRef.stdout.ref = consoleLogger +rootLogger.appenderRef.file.ref = fileLogger diff --git a/clients/build.gradle.kts b/clients/build.gradle.kts index 2d4a7344247..043fbfec673 100644 --- a/clients/build.gradle.kts +++ b/clients/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ tasks.all { diff --git a/clients/client-java-runtime/build.gradle.kts b/clients/client-java-runtime/build.gradle.kts index 1e81f51c761..d21f9d4f39a 100644 --- a/clients/client-java-runtime/build.gradle.kts +++ b/clients/client-java-runtime/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar diff --git a/clients/client-java/build.gradle.kts b/clients/client-java/build.gradle.kts index 629cec8f62c..4f38e3c3733 100644 --- a/clients/client-java/build.gradle.kts +++ b/clients/client-java/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ plugins { `maven-publish` diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/AuthDataProvider.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/AuthDataProvider.java index 1c6c686714a..c446807fe91 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/AuthDataProvider.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/AuthDataProvider.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/BaseSchemaCatalog.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/BaseSchemaCatalog.java index 903ba118ae5..593d4febbcb 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/BaseSchemaCatalog.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/BaseSchemaCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; @@ -37,14 +51,12 @@ abstract class BaseSchemaCatalog extends CatalogDTO implements Catalog, SupportsSchemas { /** The REST client to send the requests. */ protected final RESTClient restClient; - /** The namespace of current catalog, which is the metalake name. */ - protected final Namespace namespace; - /** The namespace of the schemas, which is the metalake name with catalog name. */ - protected final Namespace schemaNamespace; + /** The namespace of current catalog, which is the metalake name. */ + private final Namespace catalogNamespace; BaseSchemaCatalog( - Namespace namespace, + Namespace catalogNamespace, String name, Catalog.Type type, String provider, @@ -54,9 +66,11 @@ abstract class BaseSchemaCatalog extends CatalogDTO implements Catalog, Supports RESTClient restClient) { super(name, type, provider, comment, properties, auditDTO); this.restClient = restClient; - Namespace.checkCatalog(namespace); - this.namespace = namespace; - this.schemaNamespace = Namespace.ofSchema(namespace.level(0), name); + Namespace.check( + catalogNamespace != null && catalogNamespace.length() == 1, + "Catalog namespace must be non-null and have 1 level, the input namespace is %s", + catalogNamespace); + this.catalogNamespace = catalogNamespace; } @Override @@ -75,7 +89,7 @@ public String[] listSchemas() throws NoSuchCatalogException { EntityListResponse resp = restClient.get( - formatSchemaRequestPath(schemaNamespace), + formatSchemaRequestPath(schemaNamespace()), EntityListResponse.class, Collections.emptyMap(), ErrorHandlers.schemaErrorHandler()); @@ -104,7 +118,7 @@ public Schema createSchema(String schemaName, String comment, Map NameIdentifier.of(ident.namespace().level(2), ident.name())) + .toArray(NameIdentifier[]::new); } /** - * Load fileset metadata by {@link NameIdentifier} from the catalog. + * Load fileset metadata by {@link NameIdentifier} from the catalog, which should be a + * "schema.fileset" style. * * @param ident A fileset identifier. * @return The fileset metadata. @@ -87,13 +106,12 @@ public NameIdentifier[] listFilesets(Namespace namespace) throws NoSuchSchemaExc */ @Override public Fileset loadFileset(NameIdentifier ident) throws NoSuchFilesetException { - NameIdentifier.checkFileset(ident); + checkFilesetNameIdentifier(ident); + Namespace fullNamespace = getFilesetFullNamespace(ident.namespace()); FilesetResponse resp = restClient.get( - formatFilesetRequestPath(ident.namespace()) - + "/" - + RESTUtils.encodeString(ident.name()), + formatFilesetRequestPath(fullNamespace) + "/" + RESTUtils.encodeString(ident.name()), FilesetResponse.class, Collections.emptyMap(), ErrorHandlers.filesetErrorHandler()); @@ -110,7 +128,7 @@ public Fileset loadFileset(NameIdentifier ident) throws NoSuchFilesetException { * *

If the type of the fileset object is "EXTERNAL", the underlying storageLocation must be set. * - * @param ident A fileset identifier. + * @param ident A fileset identifier, which should be a "schema.fileset" style. * @param comment The comment of the fileset. * @param type The type of the fileset. * @param storageLocation The storage location of the fileset. @@ -127,8 +145,9 @@ public Fileset createFileset( String storageLocation, Map properties) throws NoSuchSchemaException, FilesetAlreadyExistsException { - NameIdentifier.checkFileset(ident); + checkFilesetNameIdentifier(ident); + Namespace fullNamespace = getFilesetFullNamespace(ident.namespace()); FilesetCreateRequest req = FilesetCreateRequest.builder() .name(RESTUtils.encodeString(ident.name())) @@ -140,7 +159,7 @@ public Fileset createFileset( FilesetResponse resp = restClient.post( - formatFilesetRequestPath(ident.namespace()), + formatFilesetRequestPath(fullNamespace), req, FilesetResponse.class, Collections.emptyMap(), @@ -153,7 +172,7 @@ public Fileset createFileset( /** * Update a fileset metadata in the catalog. * - * @param ident A fileset identifier. + * @param ident A fileset identifier, which should be a "schema.fileset" style. * @param changes The changes to apply to the fileset. * @return The updated fileset metadata. * @throws NoSuchFilesetException If the fileset does not exist. @@ -162,8 +181,9 @@ public Fileset createFileset( @Override public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) throws NoSuchFilesetException, IllegalArgumentException { - NameIdentifier.checkFileset(ident); + checkFilesetNameIdentifier(ident); + Namespace fullNamespace = getFilesetFullNamespace(ident.namespace()); List updates = Arrays.stream(changes) .map(DTOConverters::toFilesetUpdateRequest) @@ -173,7 +193,7 @@ public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) FilesetResponse resp = restClient.put( - formatFilesetRequestPath(ident.namespace()) + "/" + ident.name(), + formatFilesetRequestPath(fullNamespace) + "/" + ident.name(), req, FilesetResponse.class, Collections.emptyMap(), @@ -189,16 +209,17 @@ public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) *

The underlying files will be deleted if this fileset type is managed, otherwise, only the * metadata will be dropped. * - * @param ident A fileset identifier. + * @param ident A fileset identifier, which should be a "schema.fileset" style. * @return true If the fileset is dropped, false the fileset did not exist. */ @Override public boolean dropFileset(NameIdentifier ident) { - NameIdentifier.checkFileset(ident); + checkFilesetNameIdentifier(ident); + Namespace fullNamespace = getFilesetFullNamespace(ident.namespace()); DropResponse resp = restClient.delete( - formatFilesetRequestPath(ident.namespace()) + "/" + ident.name(), + formatFilesetRequestPath(fullNamespace) + "/" + ident.name(), DropResponse.class, Collections.emptyMap(), ErrorHandlers.filesetErrorHandler()); @@ -218,6 +239,34 @@ static String formatFilesetRequestPath(Namespace ns) { .toString(); } + /** + * Check whether the namespace of a fileset is valid + * + * @param namespace The namespace to check + */ + static void checkFilesetNamespace(Namespace namespace) { + Namespace.check( + namespace != null && namespace.length() == 1, + "Fileset namespace must be non-null and have 1 level, the input namespace is %s", + namespace); + } + + /** + * Check whether the NameIdentifier of a fileset is valid + * + * @param ident The NameIdentifier to check + */ + static void checkFilesetNameIdentifier(NameIdentifier ident) { + NameIdentifier.check(ident != null, "NameIdentifier must not be null"); + NameIdentifier.check( + ident.name() != null && !ident.name().isEmpty(), "NameIdentifier name must not be empty"); + checkFilesetNamespace(ident.namespace()); + } + + private Namespace getFilesetFullNamespace(Namespace tableNamespace) { + return Namespace.of(this.catalogNamespace().level(0), this.name(), tableNamespace.level(0)); + } + /** * Create a new builder for the fileset catalog. * @@ -247,7 +296,10 @@ Builder withRestClient(RESTClient restClient) { @Override public FilesetCatalog build() { - Namespace.checkCatalog(namespace); + Namespace.check( + namespace != null && namespace.length() == 1, + "Catalog namespace must be non-null and have 1 level, the input namespace is %s", + namespace); Preconditions.checkArgument(restClient != null, "restClient must be set"); Preconditions.checkArgument(StringUtils.isNotBlank(name), "name must not be blank"); Preconditions.checkArgument(type != null, "type must not be null"); diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java index aaff7ee9a56..4bf8106d81f 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoAdminClient.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClient.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClient.java index 598156d1c6d..f89f18572cb 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClient.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClient.java @@ -1,13 +1,26 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.CatalogChange; -import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.SupportsCatalogs; import com.datastrato.gravitino.exceptions.CatalogAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchCatalogException; @@ -58,7 +71,7 @@ private GravitinoMetalake getMetalake() { } @Override - public NameIdentifier[] listCatalogs() throws NoSuchMetalakeException { + public String[] listCatalogs() throws NoSuchMetalakeException { return getMetalake().listCatalogs(); } diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java index 56e8a750789..2bffa284fb6 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoClientBase.java @@ -1,11 +1,26 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.Version; import com.datastrato.gravitino.dto.responses.MetalakeResponse; import com.datastrato.gravitino.dto.responses.VersionResponse; @@ -126,7 +141,11 @@ public GravitinoMetalake loadMetalake(String metalakeName) throws NoSuchMetalake * @throws IllegalNameIdentifierException If the Metalake name is invalid. */ public void checkMetalakeName(String metalakeName) { - NameIdentifier.checkMetalake(NameIdentifier.parse(metalakeName)); + NameIdentifier identifier = NameIdentifier.parse(metalakeName); + Namespace.check( + identifier.namespace() != null && identifier.namespace().isEmpty(), + "Metalake namespace must be non-null and empty, the input namespace is %s", + identifier.namespace()); } /** diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoMetalake.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoMetalake.java index 5b8cd070230..dec10e6836b 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoMetalake.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoMetalake.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; @@ -52,11 +66,11 @@ public class GravitinoMetalake extends MetalakeDTO implements SupportsCatalogs { /** * List all the catalogs under this metalake. * - * @return A list of {@link NameIdentifier} of the catalogs under the specified namespace. - * @throws NoSuchMetalakeException if the metalake with specified namespace does not exist. + * @return A list of the catalog names under the current metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. */ @Override - public NameIdentifier[] listCatalogs() throws NoSuchMetalakeException { + public String[] listCatalogs() throws NoSuchMetalakeException { EntityListResponse resp = restClient.get( @@ -66,7 +80,7 @@ public NameIdentifier[] listCatalogs() throws NoSuchMetalakeException { ErrorHandlers.catalogErrorHandler()); resp.validate(); - return resp.identifiers(); + return Arrays.stream(resp.identifiers()).map(NameIdentifier::name).toArray(String[]::new); } /** diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoVersion.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoVersion.java index abf8a23e65d..5a984bf8c95 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoVersion.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/GravitinoVersion.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/KerberosTokenProvider.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/KerberosTokenProvider.java index 900eece91a5..0a4a1ba2664 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/KerberosTokenProvider.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/KerberosTokenProvider.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/MessagingCatalog.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/MessagingCatalog.java index 681d4319599..c322ef0d50e 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/MessagingCatalog.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/MessagingCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; @@ -63,40 +77,45 @@ public TopicCatalog asTopicCatalog() throws UnsupportedOperationException { /** * List all the topics under the given namespace. * - * @param namespace The namespace to list the topics under it. + * @param namespace A schema namespace. This namespace should have 1 level, which is the schema + * name; * @return An array of {@link NameIdentifier} of the topics under the specified namespace. * @throws NoSuchSchemaException if the schema with specified namespace does not exist. */ @Override public NameIdentifier[] listTopics(Namespace namespace) throws NoSuchSchemaException { - Namespace.checkTopic(namespace); + checkTopicNamespace(namespace); + Namespace fullNamespace = getTopicFullNamespace(namespace); EntityListResponse resp = restClient.get( - formatTopicRequestPath(namespace), + formatTopicRequestPath(fullNamespace), EntityListResponse.class, Collections.emptyMap(), ErrorHandlers.topicErrorHandler()); resp.validate(); - return resp.identifiers(); + return Arrays.stream(resp.identifiers()) + .map(ident -> NameIdentifier.of(ident.namespace().level(2), ident.name())) + .toArray(NameIdentifier[]::new); } /** * Load the topic with the given identifier. * - * @param ident The identifier of the topic to load. + * @param ident The identifier of the topic to load, which should be a "schema.topic" style. * @return The {@link Topic} with the specified identifier. * @throws NoSuchTopicException if the topic with the specified identifier does not exist. */ @Override public Topic loadTopic(NameIdentifier ident) throws NoSuchTopicException { - NameIdentifier.checkTopic(ident); + checkTopicNameIdentifier(ident); + Namespace fullNamespace = getTopicFullNamespace(ident.namespace()); TopicResponse resp = restClient.get( - formatTopicRequestPath(ident.namespace()) + "/" + ident.name(), + formatTopicRequestPath(fullNamespace) + "/" + ident.name(), TopicResponse.class, Collections.emptyMap(), ErrorHandlers.topicErrorHandler()); @@ -108,7 +127,7 @@ public Topic loadTopic(NameIdentifier ident) throws NoSuchTopicException { /** * Create a new topic with the given identifier, comment, data layout and properties. * - * @param ident A topic identifier. + * @param ident A topic identifier, which should be a "schema.topic" style. * @param comment The comment of the topic object. Null is set if no comment is specified. * @param dataLayout The message schema of the topic object. Always null because it's not * supported yet. @@ -122,8 +141,9 @@ public Topic loadTopic(NameIdentifier ident) throws NoSuchTopicException { public Topic createTopic( NameIdentifier ident, String comment, DataLayout dataLayout, Map properties) throws NoSuchSchemaException, TopicAlreadyExistsException { - NameIdentifier.checkTopic(ident); + checkTopicNameIdentifier(ident); + Namespace fullNamespace = getTopicFullNamespace(ident.namespace()); TopicCreateRequest req = TopicCreateRequest.builder() .name(ident.name()) @@ -133,7 +153,7 @@ public Topic createTopic( TopicResponse resp = restClient.post( - formatTopicRequestPath(ident.namespace()), + formatTopicRequestPath(fullNamespace), req, TopicResponse.class, Collections.emptyMap(), @@ -146,7 +166,7 @@ public Topic createTopic( /** * Alter the topic with the given identifier. * - * @param ident A topic identifier. + * @param ident A topic identifier, which should be a "schema.topic" style. * @param changes The changes to apply to the topic. * @return The altered topic object. * @throws NoSuchTopicException if the topic with the specified identifier does not exist. @@ -155,8 +175,9 @@ public Topic createTopic( @Override public Topic alterTopic(NameIdentifier ident, TopicChange... changes) throws NoSuchTopicException, IllegalArgumentException { - NameIdentifier.checkTopic(ident); + checkTopicNameIdentifier(ident); + Namespace fullNamespace = getTopicFullNamespace(ident.namespace()); List updates = Arrays.stream(changes) .map(DTOConverters::toTopicUpdateRequest) @@ -166,7 +187,7 @@ public Topic alterTopic(NameIdentifier ident, TopicChange... changes) TopicResponse resp = restClient.put( - formatTopicRequestPath(ident.namespace()) + "/" + ident.name(), + formatTopicRequestPath(fullNamespace) + "/" + ident.name(), updatesRequest, TopicResponse.class, Collections.emptyMap(), @@ -179,16 +200,17 @@ public Topic alterTopic(NameIdentifier ident, TopicChange... changes) /** * Drop the topic with the given identifier. * - * @param ident A topic identifier. + * @param ident A topic identifier, which should be a "schema.topic" style. * @return True if the topic is dropped successfully, false the topic does not exist. */ @Override public boolean dropTopic(NameIdentifier ident) { - NameIdentifier.checkTopic(ident); + checkTopicNameIdentifier(ident); + Namespace fullNamespace = getTopicFullNamespace(ident.namespace()); DropResponse resp = restClient.delete( - formatTopicRequestPath(ident.namespace()) + "/" + ident.name(), + formatTopicRequestPath(fullNamespace) + "/" + ident.name(), DropResponse.class, Collections.emptyMap(), ErrorHandlers.topicErrorHandler()); @@ -203,6 +225,34 @@ static String formatTopicRequestPath(Namespace ns) { return formatSchemaRequestPath(schemaNs) + "/" + ns.level(2) + "/topics"; } + /** + * Check whether the namespace of a topic is valid + * + * @param namespace The namespace to check + */ + static void checkTopicNamespace(Namespace namespace) { + Namespace.check( + namespace != null && namespace.length() == 1, + "Topic namespace must be non-null and have 1 level, the input namespace is %s", + namespace); + } + + /** + * Check whether the NameIdentifier of a topic is valid + * + * @param ident The NameIdentifier to check + */ + static void checkTopicNameIdentifier(NameIdentifier ident) { + NameIdentifier.check(ident != null, "NameIdentifier must not be null"); + NameIdentifier.check( + ident.name() != null && !ident.name().isEmpty(), "NameIdentifier name must not be empty"); + checkTopicNamespace(ident.namespace()); + } + + private Namespace getTopicFullNamespace(Namespace tableNamespace) { + return Namespace.of(this.catalogNamespace().level(0), this.name(), tableNamespace.level(0)); + } + static class Builder extends CatalogDTO.Builder { /** The REST client to send the requests. */ private RESTClient restClient; @@ -224,7 +274,10 @@ Builder withRestClient(RESTClient restClient) { @Override public MessagingCatalog build() { - Namespace.checkCatalog(namespace); + Namespace.check( + namespace != null && namespace.length() == 1, + "Catalog namespace must be non-null and have 1 level, the input namespace is %s", + namespace); Preconditions.checkArgument(StringUtils.isNotBlank(name), "name must not be blank"); Preconditions.checkArgument(type != null, "type must not be null"); Preconditions.checkArgument(StringUtils.isNotBlank(provider), "provider must not be blank"); diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/OAuth2TokenProvider.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/OAuth2TokenProvider.java index 540970dce88..f40ddb6b913 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/OAuth2TokenProvider.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/OAuth2TokenProvider.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/ObjectMapperProvider.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/ObjectMapperProvider.java index 0e8c65acd35..b73c9656542 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/ObjectMapperProvider.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/ObjectMapperProvider.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/RelationalCatalog.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/RelationalCatalog.java index dfceede69aa..481e98c29ef 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/RelationalCatalog.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/RelationalCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; @@ -67,51 +81,56 @@ public TableCatalog asTableCatalog() { /** * List all the tables under the given Schema namespace. * - * @param namespace The namespace to list the tables under it. + * @param namespace The namespace to list the tables under it. This namespace should have 1 level, + * which is the schema name; * @return A list of {@link NameIdentifier} of the tables under the given namespace. * @throws NoSuchSchemaException if the schema with specified namespace does not exist. */ @Override public NameIdentifier[] listTables(Namespace namespace) throws NoSuchSchemaException { - Namespace.checkTable(namespace); + checkTableNamespace(namespace); + Namespace fullNamespace = getTableFullNamespace(namespace); EntityListResponse resp = restClient.get( - formatTableRequestPath(namespace), + formatTableRequestPath(fullNamespace), EntityListResponse.class, Collections.emptyMap(), ErrorHandlers.tableErrorHandler()); resp.validate(); - return resp.identifiers(); + return Arrays.stream(resp.identifiers()) + .map(ident -> NameIdentifier.of(ident.namespace().level(2), ident.name())) + .toArray(NameIdentifier[]::new); } /** * Load the table with specified identifier. * - * @param ident The identifier of the table to load. + * @param ident The identifier of the table to load, which should be a "schema.table" style. * @return The {@link Table} with specified identifier. * @throws NoSuchTableException if the table with specified identifier does not exist. */ @Override public Table loadTable(NameIdentifier ident) throws NoSuchTableException { - NameIdentifier.checkTable(ident); + checkTableNameIdentifier(ident); + Namespace fullNamespace = getTableFullNamespace(ident.namespace()); TableResponse resp = restClient.get( - formatTableRequestPath(ident.namespace()) + "/" + RESTUtils.encodeString(ident.name()), + formatTableRequestPath(fullNamespace) + "/" + RESTUtils.encodeString(ident.name()), TableResponse.class, Collections.emptyMap(), ErrorHandlers.tableErrorHandler()); resp.validate(); - return RelationalTable.from(ident.namespace(), resp.getTable(), restClient); + return RelationalTable.from(fullNamespace, resp.getTable(), restClient); } /** * Create a new table with specified identifier, columns, comment and properties. * - * @param ident The identifier of the table. + * @param ident The identifier of the table, which should be a "schema.table" style. * @param columns The columns of the table. * @param comment The comment of the table. * @param properties The properties of the table. @@ -132,7 +151,7 @@ public Table createTable( SortOrder[] sortOrders, Index[] indexes) throws NoSuchSchemaException, TableAlreadyExistsException { - NameIdentifier.checkTable(ident); + checkTableNameIdentifier(ident); TableCreateRequest req = new TableCreateRequest( @@ -146,22 +165,23 @@ public Table createTable( toDTOs(indexes)); req.validate(); + Namespace fullNamespace = getTableFullNamespace(ident.namespace()); TableResponse resp = restClient.post( - formatTableRequestPath(ident.namespace()), + formatTableRequestPath(fullNamespace), req, TableResponse.class, Collections.emptyMap(), ErrorHandlers.tableErrorHandler()); resp.validate(); - return RelationalTable.from(ident.namespace(), resp.getTable(), restClient); + return RelationalTable.from(fullNamespace, resp.getTable(), restClient); } /** * Alter the table with specified identifier by applying the changes. * - * @param ident The identifier of the table. + * @param ident The identifier of the table, which should be a "schema.table" style. * @param changes Table changes to apply to the table. * @return The altered {@link Table}. * @throws NoSuchTableException if the table with specified identifier does not exist. @@ -170,7 +190,7 @@ public Table createTable( @Override public Table alterTable(NameIdentifier ident, TableChange... changes) throws NoSuchTableException, IllegalArgumentException { - NameIdentifier.checkTable(ident); + checkTableNameIdentifier(ident); List reqs = Arrays.stream(changes) @@ -179,31 +199,33 @@ public Table alterTable(NameIdentifier ident, TableChange... changes) TableUpdatesRequest updatesRequest = new TableUpdatesRequest(reqs); updatesRequest.validate(); + Namespace fullNamespace = getTableFullNamespace(ident.namespace()); TableResponse resp = restClient.put( - formatTableRequestPath(ident.namespace()) + "/" + RESTUtils.encodeString(ident.name()), + formatTableRequestPath(fullNamespace) + "/" + RESTUtils.encodeString(ident.name()), updatesRequest, TableResponse.class, Collections.emptyMap(), ErrorHandlers.tableErrorHandler()); resp.validate(); - return RelationalTable.from(ident.namespace(), resp.getTable(), restClient); + return RelationalTable.from(fullNamespace, resp.getTable(), restClient); } /** * Drop the table with specified identifier. * - * @param ident The identifier of the table. + * @param ident The identifier of the table, which should be a "schema.table" style. * @return true if the table is dropped successfully, false if the table does not exist. */ @Override public boolean dropTable(NameIdentifier ident) { - NameIdentifier.checkTable(ident); + checkTableNameIdentifier(ident); + Namespace fullNamespace = getTableFullNamespace(ident.namespace()); DropResponse resp = restClient.delete( - formatTableRequestPath(ident.namespace()) + "/" + RESTUtils.encodeString(ident.name()), + formatTableRequestPath(fullNamespace) + "/" + RESTUtils.encodeString(ident.name()), DropResponse.class, Collections.emptyMap(), ErrorHandlers.tableErrorHandler()); @@ -214,21 +236,20 @@ public boolean dropTable(NameIdentifier ident) { /** * Purge the table with specified identifier. * - * @param ident The identifier of the table. + * @param ident The identifier of the table, which should be a "schema.table" style. * @return true if the table is purged successfully, false otherwise. */ @Override public boolean purgeTable(NameIdentifier ident) throws UnsupportedOperationException { - NameIdentifier.checkTable(ident); + checkTableNameIdentifier(ident); + Namespace fullNamespace = getTableFullNamespace(ident.namespace()); Map params = new HashMap<>(); params.put("purge", "true"); try { DropResponse resp = restClient.delete( - formatTableRequestPath(ident.namespace()) - + "/" - + RESTUtils.encodeString(ident.name()), + formatTableRequestPath(fullNamespace) + "/" + RESTUtils.encodeString(ident.name()), params, DropResponse.class, Collections.emptyMap(), @@ -253,6 +274,40 @@ static String formatTableRequestPath(Namespace ns) { .toString(); } + /** + * Check whether the namespace of a table is valid, which should be "schema". + * + * @param namespace The namespace to check + */ + static void checkTableNamespace(Namespace namespace) { + Namespace.check( + namespace != null && namespace.length() == 1, + "Table namespace must be non-null and have 1 level, the input namespace is %s", + namespace); + } + + /** + * Check whether the NameIdentifier of a table is valid, which should be a "schema.table" style. + * + * @param ident The NameIdentifier to check + */ + static void checkTableNameIdentifier(NameIdentifier ident) { + NameIdentifier.check(ident != null, "NameIdentifier must not be null"); + NameIdentifier.check( + ident.name() != null && !ident.name().isEmpty(), "NameIdentifier name must not be empty"); + checkTableNamespace(ident.namespace()); + } + + /** + * Get the full namespace of the table with the given table's short namespace (schema name). + * + * @param tableNamespace The table's short namespace (schema name). + * @return full namespace of the table (metalake.catalog.schema). + */ + private Namespace getTableFullNamespace(Namespace tableNamespace) { + return Namespace.of(this.catalogNamespace().level(0), this.name(), tableNamespace.level(0)); + } + /** * Create a new builder for the relational catalog. * @@ -282,7 +337,10 @@ Builder withRestClient(RESTClient restClient) { @Override public RelationalCatalog build() { - Namespace.checkCatalog(namespace); + Namespace.check( + namespace != null && namespace.length() == 1, + "Catalog namespace must be non-null and have 1 level, the input namespace is %s", + namespace); Preconditions.checkArgument(restClient != null, "restClient must be set"); Preconditions.checkArgument(StringUtils.isNotBlank(name), "name must not be blank"); Preconditions.checkArgument(type != null, "type must not be null"); diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/RelationalTable.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/RelationalTable.java index 26628374b46..a9d02b6f8c3 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/RelationalTable.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/RelationalTable.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; @@ -40,7 +54,7 @@ public class RelationalTable implements Table, SupportsPartitions { /** * Creates a new RelationalTable. * - * @param namespace The namespace of the table. + * @param namespace The full namespace of the table, including metalake, catalog and schema. * @param tableDTO The table data transfer object. * @param restClient The REST client. * @return A new RelationalTable. @@ -57,25 +71,16 @@ public static RelationalTable from( /** * Creates a new RelationalTable. * - * @param namespace The namespace of the table. + * @param namespace The full namespace of the table, including metalake, catalog and schema * @param tableDTO The table data transfer object. * @param restClient The REST client. */ - public RelationalTable(Namespace namespace, TableDTO tableDTO, RESTClient restClient) { + private RelationalTable(Namespace namespace, TableDTO tableDTO, RESTClient restClient) { this.namespace = namespace; this.restClient = restClient; this.table = fromDTO(tableDTO); } - /** - * Returns the namespace of the table. - * - * @return The namespace of the table. - */ - public Namespace namespace() { - return namespace; - } - /** * Returns the name of the table. * diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/SimpleTokenProvider.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/SimpleTokenProvider.java index 8297b9df32b..7a4bcb55603 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/SimpleTokenProvider.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/SimpleTokenProvider.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestBase.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestBase.java index 3e6edb6b34a..89368f2e306 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestBase.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestBase.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestFilesetCatalog.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestFilesetCatalog.java index a82dbfba2c9..901253c4ceb 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestFilesetCatalog.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestFilesetCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; @@ -11,6 +25,7 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.CatalogDTO; import com.datastrato.gravitino.dto.file.FilesetDTO; @@ -87,11 +102,20 @@ public static void setUp() throws Exception { @Test public void testListFileset() throws JsonProcessingException { - NameIdentifier fileset1 = NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset1"); - NameIdentifier fileset2 = NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset2"); - String filesetPath = withSlash(FilesetCatalog.formatFilesetRequestPath(fileset1.namespace())); + NameIdentifier fileset1 = NameIdentifier.of("schema1", "fileset1"); + NameIdentifier fileset2 = NameIdentifier.of("schema1", "fileset2"); + NameIdentifier expectedResultFileset1 = + NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset1"); + NameIdentifier expectedResultFileset2 = + NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset2"); + String filesetPath = + withSlash( + FilesetCatalog.formatFilesetRequestPath( + Namespace.of(metalakeName, catalogName, "schema1"))); - EntityListResponse resp = new EntityListResponse(new NameIdentifier[] {fileset1, fileset2}); + EntityListResponse resp = + new EntityListResponse( + new NameIdentifier[] {expectedResultFileset1, expectedResultFileset2}); buildMockResource(Method.GET, filesetPath, null, resp, SC_OK); NameIdentifier[] filesets = catalog.asFilesetCatalog().listFilesets(fileset1.namespace()); @@ -128,9 +152,12 @@ public void testListFileset() throws JsonProcessingException { @Test public void testLoadFileset() throws JsonProcessingException { - NameIdentifier fileset = NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset1"); + NameIdentifier fileset = NameIdentifier.of("schema1", "fileset1"); String filesetPath = - withSlash(FilesetCatalog.formatFilesetRequestPath(fileset.namespace()) + "/fileset1"); + withSlash( + FilesetCatalog.formatFilesetRequestPath( + Namespace.of(metalakeName, catalogName, "schema1")) + + "/fileset1"); FilesetDTO mockFileset = mockFilesetDTO( @@ -172,8 +199,11 @@ public void testLoadFileset() throws JsonProcessingException { @Test public void testCreateFileset() throws JsonProcessingException { - NameIdentifier fileset = NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset1"); - String filesetPath = withSlash(FilesetCatalog.formatFilesetRequestPath(fileset.namespace())); + NameIdentifier fileset = NameIdentifier.of("schema1", "fileset1"); + String filesetPath = + withSlash( + FilesetCatalog.formatFilesetRequestPath( + Namespace.of(metalakeName, catalogName, "schema1"))); FilesetDTO mockFileset = mockFilesetDTO( @@ -241,9 +271,12 @@ public void testCreateFileset() throws JsonProcessingException { @Test public void testDropFileset() throws JsonProcessingException { - NameIdentifier fileset = NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset1"); + NameIdentifier fileset = NameIdentifier.of("schema1", "fileset1"); String filesetPath = - withSlash(FilesetCatalog.formatFilesetRequestPath(fileset.namespace()) + "/fileset1"); + withSlash( + FilesetCatalog.formatFilesetRequestPath( + Namespace.of(metalakeName, catalogName, "schema1")) + + "/fileset1"); DropResponse resp = new DropResponse(true); buildMockResource(Method.DELETE, filesetPath, null, resp, SC_OK); @@ -266,9 +299,12 @@ public void testDropFileset() throws JsonProcessingException { @Test public void testAlterFileset() throws JsonProcessingException { - NameIdentifier fileset = NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset1"); + NameIdentifier fileset = NameIdentifier.of("schema1", "fileset1"); String filesetPath = - withSlash(FilesetCatalog.formatFilesetRequestPath(fileset.namespace()) + "/fileset1"); + withSlash( + FilesetCatalog.formatFilesetRequestPath( + Namespace.of(metalakeName, catalogName, "schema1")) + + "/fileset1"); // Test alter fileset name FilesetUpdateRequest req = new FilesetUpdateRequest.RenameFilesetRequest("new name"); diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClient.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClient.java index 98123eb338a..c6f03ba1ba6 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClient.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClient.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClientBuilder.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClientBuilder.java index b8ba3c64a0a..972510d714c 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClientBuilder.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoClientBuilder.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoMetalake.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoMetalake.java index 1969b8fff0d..436e43cfb2e 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoMetalake.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoMetalake.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; @@ -76,16 +90,16 @@ public void testListCatalogs() throws JsonProcessingException { EntityListResponse resp = new EntityListResponse(new NameIdentifier[] {ident1, ident2}); buildMockResource(Method.GET, path, null, resp, HttpStatus.SC_OK); - NameIdentifier[] catalogs = gravitinoClient.listCatalogs(); + String[] catalogs = gravitinoClient.listCatalogs(); Assertions.assertEquals(2, catalogs.length); - Assertions.assertEquals(ident1, catalogs[0]); - Assertions.assertEquals(ident2, catalogs[1]); + Assertions.assertEquals(ident1.name(), catalogs[0]); + Assertions.assertEquals(ident2.name(), catalogs[1]); // Test return empty catalog list EntityListResponse resp1 = new EntityListResponse(new NameIdentifier[] {}); buildMockResource(Method.GET, path, null, resp1, HttpStatus.SC_OK); - NameIdentifier[] catalogs1 = gravitinoClient.listCatalogs(); + String[] catalogs1 = gravitinoClient.listCatalogs(); Assertions.assertEquals(0, catalogs1.length); // Test return internal error diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoVersion.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoVersion.java index c94b613af67..9d4dcfaf397 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoVersion.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestGravitinoVersion.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestMessagingCatalog.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestMessagingCatalog.java index 0981e1d4881..5a419db4d09 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestMessagingCatalog.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestMessagingCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; @@ -11,6 +25,7 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.CatalogDTO; import com.datastrato.gravitino.dto.messaging.TopicDTO; @@ -96,11 +111,13 @@ public void testListTopics() throws Exception { EntityListResponse entityListResponse = new EntityListResponse(new NameIdentifier[] {topic1, topic2}); buildMockResource(Method.GET, topicPath, null, entityListResponse, SC_OK); - NameIdentifier[] topics = ((MessagingCatalog) catalog).listTopics(topic1.namespace()); + NameIdentifier[] topics = ((MessagingCatalog) catalog).listTopics(Namespace.of("schema1")); + NameIdentifier expectedResultTopic1 = NameIdentifier.of("schema1", "topic1"); + NameIdentifier expectedResultTopic2 = NameIdentifier.of("schema1", "topic2"); Assertions.assertEquals(2, topics.length); - Assertions.assertEquals(topic1, topics[0]); - Assertions.assertEquals(topic2, topics[1]); + Assertions.assertEquals(expectedResultTopic1, topics[0]); + Assertions.assertEquals(expectedResultTopic2, topics[1]); // Throw schema not found exception ErrorResponse errResp = @@ -108,7 +125,7 @@ public void testListTopics() throws Exception { buildMockResource(Method.GET, topicPath, null, errResp, SC_NOT_FOUND); Assertions.assertThrows( NoSuchSchemaException.class, - () -> catalog.asTopicCatalog().listTopics(topic1.namespace()), + () -> catalog.asTopicCatalog().listTopics(expectedResultTopic1.namespace()), "schema not found"); // Throw Runtime exception @@ -116,15 +133,19 @@ public void testListTopics() throws Exception { buildMockResource(Method.GET, topicPath, null, errResp2, SC_SERVER_ERROR); Assertions.assertThrows( RuntimeException.class, - () -> catalog.asTopicCatalog().listTopics(topic1.namespace()), + () -> catalog.asTopicCatalog().listTopics(expectedResultTopic1.namespace()), "internal error"); } @Test public void testLoadTopic() throws JsonProcessingException { - NameIdentifier topic = NameIdentifier.of(metalakeName, catalogName, "schema1", "topic1"); + NameIdentifier topic = NameIdentifier.of("schema1", "topic1"); String topicPath = - withSlash(MessagingCatalog.formatTopicRequestPath(topic.namespace()) + "/" + topic.name()); + withSlash( + MessagingCatalog.formatTopicRequestPath( + Namespace.of(metalakeName, catalogName, "schema1")) + + "/" + + topic.name()); TopicDTO mockTopic = mockTopicDTO(topic.name(), "comment", ImmutableMap.of("k1", "k2")); TopicResponse topicResponse = new TopicResponse(mockTopic); @@ -146,8 +167,11 @@ public void testLoadTopic() throws JsonProcessingException { @Test public void testCreateTopic() throws JsonProcessingException { - NameIdentifier topic = NameIdentifier.of(metalakeName, catalogName, "schema1", "topic1"); - String topicPath = withSlash(MessagingCatalog.formatTopicRequestPath(topic.namespace())); + NameIdentifier topic = NameIdentifier.of("schema1", "topic1"); + String topicPath = + withSlash( + MessagingCatalog.formatTopicRequestPath( + Namespace.of(metalakeName, catalogName, "schema1"))); TopicDTO mockTopic = mockTopicDTO(topic.name(), "comment", ImmutableMap.of("k1", "k2")); @@ -204,9 +228,13 @@ public void testCreateTopic() throws JsonProcessingException { @Test public void testAlterTopic() throws JsonProcessingException { - NameIdentifier topic = NameIdentifier.of(metalakeName, catalogName, "schema1", "topic1"); + NameIdentifier topic = NameIdentifier.of("schema1", "topic1"); String topicPath = - withSlash(MessagingCatalog.formatTopicRequestPath(topic.namespace()) + "/" + topic.name()); + withSlash( + MessagingCatalog.formatTopicRequestPath( + Namespace.of(metalakeName, catalogName, "schema1")) + + "/" + + topic.name()); // test alter topic comment TopicUpdateRequest req1 = new TopicUpdateRequest.UpdateTopicCommentRequest("new comment"); @@ -271,9 +299,13 @@ public void testAlterTopic() throws JsonProcessingException { @Test public void testDropTopic() throws JsonProcessingException { - NameIdentifier topic = NameIdentifier.of(metalakeName, catalogName, "schema1", "topic1"); + NameIdentifier topic = NameIdentifier.of("schema1", "topic1"); String topicPath = - withSlash(MessagingCatalog.formatTopicRequestPath(topic.namespace()) + "/" + topic.name()); + withSlash( + MessagingCatalog.formatTopicRequestPath( + Namespace.of(metalakeName, catalogName, "schema1")) + + "/" + + topic.name()); DropResponse resp = new DropResponse(true); buildMockResource(Method.DELETE, topicPath, null, resp, SC_OK); diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestMetalakeAdmin.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestMetalakeAdmin.java index 66096b7f7b5..af5d364958b 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestMetalakeAdmin.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestMetalakeAdmin.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestOAuth2TokenProvider.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestOAuth2TokenProvider.java index c612056834b..5322d49d520 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestOAuth2TokenProvider.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestOAuth2TokenProvider.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestPermission.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestPermission.java index 69ac63c1ccf..6443b37c655 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestPermission.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestPermission.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java index a00ffd1e040..87781850c1d 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; @@ -176,17 +190,20 @@ public void testListSchemas() throws JsonProcessingException { @Test public void testCreateSchema() throws JsonProcessingException { - NameIdentifier schemaId = NameIdentifier.of(metalakeName, catalogName, "schema1"); - String schemaPath = withSlash(RelationalCatalog.formatSchemaRequestPath(schemaId.namespace())); - SchemaDTO schema = createMockSchema("schema1", "comment", Collections.emptyMap()); + String schemaName = "schema1"; + String schemaPath = + withSlash( + RelationalCatalog.formatSchemaRequestPath(Namespace.of(metalakeName, catalogName))); + SchemaDTO schema = createMockSchema(schemaName, "comment", Collections.emptyMap()); - SchemaCreateRequest req = new SchemaCreateRequest("schema1", "comment", Collections.emptyMap()); + SchemaCreateRequest req = + new SchemaCreateRequest(schemaName, "comment", Collections.emptyMap()); SchemaResponse resp = new SchemaResponse(schema); buildMockResource(Method.POST, schemaPath, req, resp, SC_OK); Schema createdSchema = - catalog.asSchemas().createSchema(schemaId.name(), "comment", Collections.emptyMap()); - Assertions.assertEquals("schema1", createdSchema.name()); + catalog.asSchemas().createSchema(schemaName, "comment", Collections.emptyMap()); + Assertions.assertEquals(schemaName, createdSchema.name()); Assertions.assertEquals("comment", createdSchema.comment()); Assertions.assertEquals(Collections.emptyMap(), createdSchema.properties()); @@ -200,7 +217,7 @@ public void testCreateSchema() throws JsonProcessingException { Throwable ex = Assertions.assertThrows( NoSuchCatalogException.class, - () -> schemas.createSchema(schemaId.name(), "comment", emptyMap)); + () -> schemas.createSchema(schemaName, "comment", emptyMap)); Assertions.assertTrue(ex.getMessage().contains("catalog not found")); // Test throw SchemaAlreadyExistsException @@ -212,16 +229,16 @@ public void testCreateSchema() throws JsonProcessingException { Throwable ex1 = Assertions.assertThrows( SchemaAlreadyExistsException.class, - () -> schemas.createSchema(schemaId.name(), "comment", emptyMap)); + () -> schemas.createSchema(schemaName, "comment", emptyMap)); Assertions.assertTrue(ex1.getMessage().contains("schema already exists")); } @Test public void testLoadSchema() throws JsonProcessingException { - NameIdentifier schemaId = NameIdentifier.of(metalakeName, catalogName, "schema1"); + NameIdentifier schemaId = NameIdentifier.of("schema1"); String schemaPath = withSlash( - RelationalCatalog.formatSchemaRequestPath(schemaId.namespace()) + RelationalCatalog.formatSchemaRequestPath(Namespace.of(metalakeName, catalogName)) + "/" + schemaId.name()); SchemaDTO schema = createMockSchema("schema1", "comment", Collections.emptyMap()); @@ -306,11 +323,14 @@ public void testListTables() throws JsonProcessingException { EntityListResponse resp = new EntityListResponse(new NameIdentifier[] {table1, table2}); buildMockResource(Method.GET, tablePath, null, resp, SC_OK); - NameIdentifier[] tables = catalog.asTableCatalog().listTables(table1.namespace()); + NameIdentifier[] tables = catalog.asTableCatalog().listTables(Namespace.of("schema1")); Assertions.assertEquals(2, tables.length); - Assertions.assertEquals(table1, tables[0]); - Assertions.assertEquals(table2, tables[1]); + + NameIdentifier expectedResult1 = NameIdentifier.of("schema1", "table1"); + NameIdentifier expectedResult2 = NameIdentifier.of("schema1", "table2"); + Assertions.assertEquals(expectedResult1, tables[0]); + Assertions.assertEquals(expectedResult2, tables[1]); // Test throw NoSuchSchemaException ErrorResponse errorResp = @@ -318,7 +338,7 @@ public void testListTables() throws JsonProcessingException { buildMockResource(Method.GET, tablePath, null, errorResp, SC_NOT_FOUND); TableCatalog tableCatalog = catalog.asTableCatalog(); - Namespace namespace1 = table1.namespace(); + Namespace namespace1 = Namespace.of("schema1"); Throwable ex = Assertions.assertThrows( NoSuchSchemaException.class, () -> tableCatalog.listTables(namespace1)); @@ -341,8 +361,9 @@ public void testListTables() throws JsonProcessingException { @Test public void testCreateTable() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); - String tablePath = withSlash(RelationalCatalog.formatTableRequestPath(tableId.namespace())); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); + Namespace fullNamespace = Namespace.of(metalakeName, catalogName, tableId.namespace().level(0)); + String tablePath = withSlash(RelationalCatalog.formatTableRequestPath(fullNamespace)); ColumnDTO[] columns = new ColumnDTO[] { @@ -449,8 +470,9 @@ public void testCreateTable() throws JsonProcessingException { @Test public void testCreatePartitionedTable() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); - String tablePath = withSlash(RelationalCatalog.formatTableRequestPath(tableId.namespace())); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); + Namespace fullNamespace = Namespace.of(metalakeName, catalogName, tableId.namespace().level(0)); + String tablePath = withSlash(RelationalCatalog.formatTableRequestPath(fullNamespace)); ColumnDTO[] columns = new ColumnDTO[] { @@ -616,8 +638,9 @@ public void testCreatePartitionedTable() throws JsonProcessingException { @Test public void testCreateIndexTable() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "index1"); - String tablePath = withSlash(RelationalCatalog.formatTableRequestPath(tableId.namespace())); + NameIdentifier tableId = NameIdentifier.of("schema1", "index1"); + Namespace fullNamespace = Namespace.of(metalakeName, catalogName, tableId.namespace().level(0)); + String tablePath = withSlash(RelationalCatalog.formatTableRequestPath(fullNamespace)); ColumnDTO[] columns = new ColumnDTO[] { @@ -713,10 +736,10 @@ private void assertTableEquals(Table expected, Table actual) { @Test public void testLoadTable() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); + Namespace fullNamespace = Namespace.of(metalakeName, catalogName, tableId.namespace().level(0)); String tablePath = - withSlash( - RelationalCatalog.formatTableRequestPath(tableId.namespace()) + "/" + tableId.name()); + withSlash(RelationalCatalog.formatTableRequestPath(fullNamespace) + "/" + tableId.name()); ColumnDTO[] columns = new ColumnDTO[] { createMockColumn("col1", Types.ByteType.get(), "comment1"), @@ -755,7 +778,7 @@ public void testLoadTable() throws JsonProcessingException { @Test public void testRenameTable() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] {createMockColumn("col1", Types.ByteType.get(), "comment1")}; @@ -779,7 +802,7 @@ public void testRenameTable() throws JsonProcessingException { @Test public void testUpdateTableComment() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] {createMockColumn("col1", Types.ByteType.get(), "comment1")}; @@ -803,7 +826,7 @@ public void testUpdateTableComment() throws JsonProcessingException { @Test public void testSetTableProperty() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] {createMockColumn("col1", Types.ByteType.get(), "comment1")}; Map properties = ImmutableMap.of("k1", "v1"); @@ -828,7 +851,7 @@ public void testSetTableProperty() throws JsonProcessingException { @Test public void testRemoveTableProperty() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] {createMockColumn("col1", Types.ByteType.get(), "comment1")}; @@ -851,7 +874,7 @@ public void testRemoveTableProperty() throws JsonProcessingException { @Test public void testAddTableColumn() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] { createMockColumn("col1", Types.ByteType.get(), "comment1"), @@ -886,7 +909,7 @@ public void testAddTableColumn() throws JsonProcessingException { @Test public void testRenameTableColumn() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] { createMockColumn("col1", Types.ByteType.get(), "comment1"), @@ -914,7 +937,7 @@ public void testRenameTableColumn() throws JsonProcessingException { @Test public void testUpdateTableColumnComment() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] {createMockColumn("col1", Types.ByteType.get(), "comment2")}; @@ -938,7 +961,7 @@ public void testUpdateTableColumnComment() throws JsonProcessingException { @Test public void testUpdateTableColumnDataType() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] {createMockColumn("col1", Types.StringType.get(), "comment1")}; @@ -962,7 +985,7 @@ public void testUpdateTableColumnDataType() throws JsonProcessingException { @Test public void testUpdateTableColumnNullability() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] {createMockColumn("col1", Types.StringType.get(), "comment1")}; @@ -985,7 +1008,7 @@ public void testUpdateTableColumnNullability() throws JsonProcessingException { @Test public void testUpdateTableColumnPosition() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] { createMockColumn("col1", Types.ByteType.get(), "comment1"), @@ -1032,7 +1055,7 @@ private SortOrderDTO[] createMockSortOrderDTO(String columnName, SortDirection d @Test public void testDeleteTableColumn() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); ColumnDTO[] columns = new ColumnDTO[] {createMockColumn("col2", Types.StringType.get(), "comment2")}; @@ -1055,10 +1078,10 @@ public void testDeleteTableColumn() throws JsonProcessingException { @Test public void testDropTable() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); + Namespace fullNamespace = Namespace.of(metalakeName, catalogName, tableId.namespace().level(0)); String tablePath = - withSlash( - RelationalCatalog.formatTableRequestPath(tableId.namespace()) + "/" + tableId.name()); + withSlash(RelationalCatalog.formatTableRequestPath(fullNamespace) + "/" + tableId.name()); DropResponse resp = new DropResponse(true); buildMockResource(Method.DELETE, tablePath, null, resp, SC_OK); @@ -1081,10 +1104,10 @@ public void testDropTable() throws JsonProcessingException { @Test public void testPurgeTable() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); + Namespace fullNamespace = Namespace.of(metalakeName, catalogName, tableId.namespace().level(0)); String tablePath = - withSlash( - RelationalCatalog.formatTableRequestPath(tableId.namespace()) + "/" + tableId.name()); + withSlash(RelationalCatalog.formatTableRequestPath(fullNamespace) + "/" + tableId.name()); DropResponse resp = new DropResponse(true); buildMockResource(Method.DELETE, tablePath, null, resp, SC_OK); @@ -1104,10 +1127,10 @@ public void testPurgeTable() throws JsonProcessingException { @Test public void testPurgeExternalTable() throws JsonProcessingException { - NameIdentifier tableId = NameIdentifier.of(metalakeName, catalogName, "schema1", "table1"); + NameIdentifier tableId = NameIdentifier.of("schema1", "table1"); + Namespace fullNamespace = Namespace.of(metalakeName, catalogName, tableId.namespace().level(0)); String tablePath = - withSlash( - RelationalCatalog.formatTableRequestPath(tableId.namespace()) + "/" + tableId.name()); + withSlash(RelationalCatalog.formatTableRequestPath(fullNamespace) + "/" + tableId.name()); DropResponse resp = new DropResponse(true); buildMockResource(Method.DELETE, tablePath, null, resp, SC_OK); @@ -1131,8 +1154,9 @@ public void testPurgeExternalTable() throws JsonProcessingException { private void testAlterTable(NameIdentifier ident, TableUpdateRequest req, TableDTO updatedTable) throws JsonProcessingException { + Namespace fullNamespace = Namespace.of(metalakeName, catalogName, ident.namespace().level(0)); String tablePath = - withSlash(RelationalCatalog.formatTableRequestPath(ident.namespace()) + "/" + ident.name()); + withSlash(RelationalCatalog.formatTableRequestPath(fullNamespace) + "/" + ident.name()); TableUpdatesRequest updatesRequest = new TableUpdatesRequest(ImmutableList.of(req)); TableResponse resp = new TableResponse(updatedTable); buildMockResource(Method.PUT, tablePath, updatesRequest, resp, SC_OK); diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalTable.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalTable.java index 55b0ed85b76..b2d7a6baffd 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalTable.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalTable.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; @@ -12,6 +26,7 @@ import static org.apache.http.HttpStatus.SC_NOT_IMPLEMENTED; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.dto.SchemaDTO; import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.dto.rel.DistributionDTO; @@ -63,8 +78,9 @@ public static void setUp() throws Exception { TestRelationalCatalog.setUp(); // setup schema - NameIdentifier schemaId = NameIdentifier.of(metalakeName, catalogName, schemaName); - String schemaPath = withSlash(RelationalCatalog.formatSchemaRequestPath(schemaId.namespace())); + String schemaPath = + withSlash( + RelationalCatalog.formatSchemaRequestPath(Namespace.of(metalakeName, catalogName))); SchemaDTO mockedSchema = createMockSchema(schemaName, "comment", Collections.emptyMap()); SchemaCreateRequest req = @@ -72,12 +88,14 @@ public static void setUp() throws Exception { SchemaResponse resp = new SchemaResponse(mockedSchema); buildMockResource(Method.POST, schemaPath, req, resp, SC_OK); - catalog.asSchemas().createSchema(schemaId.name(), "comment", Collections.emptyMap()); + catalog.asSchemas().createSchema(schemaName, "comment", Collections.emptyMap()); // setup partitioned table - NameIdentifier tableId = - NameIdentifier.of(metalakeName, catalogName, schemaName, partitionedTableName); - String tablePath = withSlash(RelationalCatalog.formatTableRequestPath(tableId.namespace())); + NameIdentifier tableId = NameIdentifier.of(schemaName, partitionedTableName); + String tablePath = + withSlash( + RelationalCatalog.formatTableRequestPath( + Namespace.of(metalakeName, catalogName, schemaName))); ColumnDTO[] columns = new ColumnDTO[] { diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRole.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRole.java index 8af1dd94583..55dfeb56269 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRole.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRole.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestSimpleTokenProvider.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestSimpleTokenProvider.java index 3ce7aed9e73..afefd477054 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestSimpleTokenProvider.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestSimpleTokenProvider.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestUserGroup.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestUserGroup.java index beb37fdf81e..03ac228aadc 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestUserGroup.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestUserGroup.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.client; diff --git a/clients/client-java/src/test/java/resources/log4j2.properties b/clients/client-java/src/test/java/resources/log4j2.properties index b9e706603a0..f3155e4c00b 100644 --- a/clients/client-java/src/test/java/resources/log4j2.properties +++ b/clients/client-java/src/test/java/resources/log4j2.properties @@ -1,6 +1,20 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # Set to debug or trace if log4j initialization is failing diff --git a/clients/client-java/src/test/resources/log4j2.properties b/clients/client-java/src/test/resources/log4j2.properties index b9e706603a0..f3155e4c00b 100644 --- a/clients/client-java/src/test/resources/log4j2.properties +++ b/clients/client-java/src/test/resources/log4j2.properties @@ -1,6 +1,20 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # Set to debug or trace if log4j initialization is failing diff --git a/clients/client-python/.gitignore b/clients/client-python/.gitignore index 3d84a1cad05..6fd78c88127 100644 --- a/clients/client-python/.gitignore +++ b/clients/client-python/.gitignore @@ -1,5 +1,20 @@ -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + **/.pytest_cache/** **/__pycache__/** @@ -12,6 +27,8 @@ venv dist build README.md +gravitino/version.ini +docs # Unit test / coverage reports htmlcov/ @@ -26,4 +43,4 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ -cover/ \ No newline at end of file +cover/ diff --git a/clients/client-python/MANIFEST.in b/clients/client-python/MANIFEST.in index edc31de3fbe..d49d96e91ab 100644 --- a/clients/client-python/MANIFEST.in +++ b/clients/client-python/MANIFEST.in @@ -1,6 +1,22 @@ -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + include requirements.txt include requirements-dev.txt -include README.md \ No newline at end of file +include README.md +include gravitino/version.ini diff --git a/clients/client-python/build.gradle.kts b/clients/client-python/build.gradle.kts index 294dca9094c..68cc897e595 100644 --- a/clients/client-python/build.gradle.kts +++ b/clients/client-python/build.gradle.kts @@ -1,8 +1,24 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ import io.github.piyushroshan.python.VenvTask +import java.net.HttpURLConnection +import java.net.URL plugins { id("io.github.piyushroshan.python-gradle-miniforge-plugin") version "1.0.0" @@ -21,11 +37,52 @@ fun deleteCacheDir(targetDir: String) { } } + +fun waitForServerIsReady(host: String = "http://localhost", port: Int = 8090, timeout: Long = 30000) { + val startTime = System.currentTimeMillis() + var exception: java.lang.Exception? + val urlString = "$host:$port/metrics" + val successPattern = Regex("\"version\"\\s*:") + + while (true) { + try { + val url = URL(urlString) + val connection = url.openConnection() as HttpURLConnection + connection.requestMethod = "GET" + connection.connectTimeout = 1000 + connection.readTimeout = 1000 + + val responseCode = connection.responseCode + if (responseCode == 200) { + val response = connection.inputStream.bufferedReader().use { it.readText() } + if (successPattern.containsMatchIn(response)) { + return // If this succeeds, the API is up and running + } else { + exception = RuntimeException("API returned unexpected response: $response") + } + } else { + exception = RuntimeException("Received non-200 response code: $responseCode") + } + } catch (e: Exception) { + // API is not available yet, continue to wait + exception = e + } + + if (System.currentTimeMillis() - startTime > timeout) { + throw RuntimeException("Timed out waiting for API to be available", exception) + } + Thread.sleep(500) // Wait for 0.5 second before checking again + } +} + fun gravitinoServer(operation: String) { val process = ProcessBuilder("${project.rootDir.path}/distribution/package/bin/gravitino.sh", operation).start() val exitCode = process.waitFor() if (exitCode == 0) { val currentContext = process.inputStream.bufferedReader().readText() + if (operation == "start") { + waitForServerIsReady() + } println("Gravitino server status: $currentContext") } else { println("Gravitino server execution failed with exit code $exitCode") @@ -100,14 +157,14 @@ tasks { val black by registering(VenvTask::class) { dependsOn(pipInstall) venvExec = "black" - args = listOf("./gravitino", "./tests") + args = listOf("./gravitino", "./tests", "./scripts") } val pylint by registering(VenvTask::class) { dependsOn(pipInstall) mustRunAfter(black) venvExec = "pylint" - args = listOf("./gravitino", "./tests") + args = listOf("./gravitino", "./tests", "./scripts") } val integrationCoverageReport by registering(VenvTask::class){ @@ -152,26 +209,41 @@ tasks { } val test by registering(VenvTask::class) { - dependsOn(pipInstall, pylint, unitTests) - + val skipUTs = project.hasProperty("skipTests") val skipPyClientITs = project.hasProperty("skipPyClientITs") val skipITs = project.hasProperty("skipITs") - if (!skipITs && !skipPyClientITs) { - dependsOn(integrationTest) + val skipAllTests = skipUTs && (skipITs || skipPyClientITs) + if (!skipAllTests) { + dependsOn(pipInstall, pylint) + if (!skipUTs) { + dependsOn(unitTests) + } + if (!skipITs && !skipPyClientITs) { + dependsOn(integrationTest) + } } } val build by registering(VenvTask::class) { + dependsOn(pylint) + venvExec = "python" + args = listOf("scripts/generate_version.py") + } + + val pydoc by registering(VenvTask::class) { + venvExec = "python" + args = listOf("scripts/generate_doc.py") } val distribution by registering(VenvTask::class) { + dependsOn(build) doFirst { delete("README.md") generatePypiProjectHomePage() delete("dist") } - venvExec = "Python3" + venvExec = "python" args = listOf("setup.py", "sdist") doLast { @@ -190,6 +262,8 @@ tasks { val clean by registering(Delete::class) { delete("build") delete("dist") + delete("docs") + delete("gravitino/version.ini") delete("gravitino.egg-info") delete("tests/unittests/htmlcov") delete("tests/unittests/.coverage") @@ -206,6 +280,6 @@ tasks { it.name.endsWith("envSetup") }.all { // add install package and code formatting before any tasks - finalizedBy(pipInstall, black, pylint) + finalizedBy(pipInstall, black, pylint, build) } } diff --git a/clients/client-python/gravitino/__init__.py b/clients/client-python/gravitino/__init__.py index 24db62a279b..e2eba1533f1 100644 --- a/clients/client-python/gravitino/__init__.py +++ b/clients/client-python/gravitino/__init__.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from gravitino.api.catalog import Catalog @@ -13,3 +27,4 @@ from gravitino.client.gravitino_admin_client import GravitinoAdminClient from gravitino.client.gravitino_metalake import GravitinoMetalake from gravitino.name_identifier import NameIdentifier +from gravitino.filesystem import gvfs diff --git a/clients/client-python/gravitino/api/__init__.py b/clients/client-python/gravitino/api/__init__.py index 5779a3ad252..c206137f175 100644 --- a/clients/client-python/gravitino/api/__init__.py +++ b/clients/client-python/gravitino/api/__init__.py @@ -1,4 +1,18 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ diff --git a/clients/client-python/gravitino/api/audit.py b/clients/client-python/gravitino/api/audit.py index 9483a99d7c4..bd534d76061 100644 --- a/clients/client-python/gravitino/api/audit.py +++ b/clients/client-python/gravitino/api/audit.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import ABC, abstractmethod diff --git a/clients/client-python/gravitino/api/auditable.py b/clients/client-python/gravitino/api/auditable.py index 2c7e51f247f..57d2174e7b0 100644 --- a/clients/client-python/gravitino/api/auditable.py +++ b/clients/client-python/gravitino/api/auditable.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import ABC, abstractmethod diff --git a/clients/client-python/gravitino/api/catalog.py b/clients/client-python/gravitino/api/catalog.py index 95134f0c631..439a4909c76 100644 --- a/clients/client-python/gravitino/api/catalog.py +++ b/clients/client-python/gravitino/api/catalog.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import abstractmethod diff --git a/clients/client-python/gravitino/api/catalog_change.py b/clients/client-python/gravitino/api/catalog_change.py index 1b51c1c6810..89121d4ef83 100644 --- a/clients/client-python/gravitino/api/catalog_change.py +++ b/clients/client-python/gravitino/api/catalog_change.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import ABC @@ -65,7 +79,7 @@ class RenameCatalog: """A catalog change to rename the catalog.""" def __init__(self, new_name): - self.new_name = new_name + self._new_name = new_name def new_name(self): """Retrieves the new name set for the catalog. @@ -73,7 +87,7 @@ def new_name(self): Returns: The new name of the catalog. """ - return self.new_name + return self._new_name def __eq__(self, other) -> bool: """Compares this RenameCatalog instance with another object for equality. Two instances are @@ -87,7 +101,7 @@ def __eq__(self, other) -> bool: """ if not isinstance(other, CatalogChange.RenameCatalog): return False - return self.new_name == other.new_name + return self.new_name() == other.new_name() def __hash__(self): """Generates a hash code for this RenameCatalog instance. The hash code is primarily based on @@ -96,7 +110,7 @@ def __hash__(self): Returns: A hash code value for this renaming operation. """ - return hash(self.new_name) + return hash(self.new_name()) def __str__(self): """Provides a string representation of the RenameCatalog instance. This string includes the @@ -105,13 +119,13 @@ class name followed by the new name of the catalog. Returns: A string summary of this renaming operation. """ - return f"RENAMECATALOG {self.new_name}" + return f"RENAMECATALOG {self.new_name()}" class UpdateCatalogComment: """A catalog change to update the catalog comment.""" def __init__(self, new_comment): - self.new_comment = new_comment + self._new_comment = new_comment def new_comment(self): """Retrieves the new comment intended for the catalog. @@ -119,7 +133,7 @@ def new_comment(self): Returns: The new comment that has been set for the catalog. """ - return self.new_comment + return self._new_comment def __eq__(self, other) -> bool: """Compares this UpdateCatalogComment instance with another object for equality. @@ -133,7 +147,7 @@ def __eq__(self, other) -> bool: """ if not isinstance(other, CatalogChange.UpdateCatalogComment): return False - return self.new_comment == other.new_comment + return self.new_comment() == other.new_comment() def __hash__(self): """Generates a hash code for this UpdateCatalogComment instance. @@ -142,7 +156,7 @@ def __hash__(self): Returns: A hash code representing this comment update operation. """ - return hash(self.new_comment) + return hash(self.new_comment()) def __str__(self): """Provides a string representation of the UpdateCatalogComment instance. @@ -151,14 +165,14 @@ def __str__(self): Returns: A string summary of this comment update operation. """ - return f"UPDATECATALOGCOMMENT {self.new_comment}" + return f"UPDATECATALOGCOMMENT {self.new_comment()}" class SetProperty: """A catalog change to set the property and value for the catalog.""" def __init__(self, catalog_property, value): - self.property = catalog_property - self.value = value + self._property = catalog_property + self._value = value def property(self): """Retrieves the name of the property being set in the catalog. @@ -166,7 +180,7 @@ def property(self): Returns: The name of the property. """ - return self.property + return self._property def value(self): """Retrieves the value assigned to the property in the catalog. @@ -174,7 +188,7 @@ def value(self): Returns: The value of the property. """ - return self.value + return self._value def __eq__(self, other) -> bool: """Compares this SetProperty instance with another object for equality. @@ -188,7 +202,7 @@ def __eq__(self, other) -> bool: """ if not isinstance(other, CatalogChange.SetProperty): return False - return self.property == other.property and self.value == other.value + return self.property() == other.property() and self.value() == other.value() def __hash__(self): """Generates a hash code for this SetProperty instance. @@ -197,7 +211,7 @@ def __hash__(self): Returns: A hash code value for this property setting. """ - return hash((self.property, self.value)) + return hash((self.property(), self.value())) def __str__(self): """Provides a string representation of the SetProperty instance. @@ -206,7 +220,7 @@ def __str__(self): Returns: A string summary of the property setting. """ - return f"SETPROPERTY {self.property} {self.value}" + return f"SETPROPERTY {self.property()} {self.value()}" class RemoveProperty: """A catalog change to remove a property from the catalog.""" diff --git a/clients/client-python/gravitino/api/fileset.py b/clients/client-python/gravitino/api/fileset.py index a007782b47f..2f343209729 100644 --- a/clients/client-python/gravitino/api/fileset.py +++ b/clients/client-python/gravitino/api/fileset.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import abstractmethod diff --git a/clients/client-python/gravitino/api/fileset_change.py b/clients/client-python/gravitino/api/fileset_change.py index 6e70eb0321f..4dba4ec9314 100644 --- a/clients/client-python/gravitino/api/fileset_change.py +++ b/clients/client-python/gravitino/api/fileset_change.py @@ -1,10 +1,24 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import ABC -from dataclasses import field +from dataclasses import dataclass, field from dataclasses_json import config @@ -63,14 +77,12 @@ def remove_property(fileset_property): """ return FilesetChange.RemoveProperty(fileset_property) + @dataclass class RenameFileset: """A fileset change to rename the fileset.""" _new_name: str = field(metadata=config(field_name="new_name")) - def __init__(self, new_name): - self._new_name = new_name - def new_name(self): """Retrieves the new name set for the fileset. @@ -111,14 +123,12 @@ def __str__(self): """ return f"RENAMEFILESET {self._new_name}" + @dataclass class UpdateFilesetComment: """A fileset change to update the fileset comment.""" _new_comment: str = field(metadata=config(field_name="new_comment")) - def __init__(self, new_comment): - self._new_comment = new_comment - def new_comment(self): """Retrieves the new comment intended for the fileset. @@ -159,16 +169,13 @@ def __str__(self): """ return f"UPDATEFILESETCOMMENT {self._new_comment}" + @dataclass class SetProperty: """A fileset change to set the property and value for the fileset.""" _property: str = field(metadata=config(field_name="property")) _value: str = field(metadata=config(field_name="value")) - def __init__(self, fileset_property: str, value: str): - self._property = fileset_property - self._value = value - def property(self): """Retrieves the name of the property being set in the fileset. @@ -217,14 +224,12 @@ def __str__(self): """ return f"SETPROPERTY {self._property} {self._value}" + @dataclass class RemoveProperty: """A fileset change to remove a property from the fileset.""" _property: str = field(metadata=config(field_name="property")) - def __init__(self, fileset_property: str): - self._property = fileset_property - def property(self): """Retrieves the name of the property to be removed from the fileset. diff --git a/clients/client-python/gravitino/api/metalake.py b/clients/client-python/gravitino/api/metalake.py index a1e996f6d8b..3f3708cbb9d 100644 --- a/clients/client-python/gravitino/api/metalake.py +++ b/clients/client-python/gravitino/api/metalake.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import abstractmethod diff --git a/clients/client-python/gravitino/api/metalake_change.py b/clients/client-python/gravitino/api/metalake_change.py index 27b91bd90dd..a34c0f0e960 100644 --- a/clients/client-python/gravitino/api/metalake_change.py +++ b/clients/client-python/gravitino/api/metalake_change.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/api/schema.py b/clients/client-python/gravitino/api/schema.py index 204abf17d01..4723a078e14 100644 --- a/clients/client-python/gravitino/api/schema.py +++ b/clients/client-python/gravitino/api/schema.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import abstractmethod diff --git a/clients/client-python/gravitino/api/schema_change.py b/clients/client-python/gravitino/api/schema_change.py index 3dd05803036..bdc33914fd7 100644 --- a/clients/client-python/gravitino/api/schema_change.py +++ b/clients/client-python/gravitino/api/schema_change.py @@ -1,10 +1,24 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import ABC -from dataclasses import field +from dataclasses import dataclass, field from dataclasses_json import config @@ -37,16 +51,13 @@ def remove_property(schema_property: str): """ return SchemaChange.RemoveProperty(schema_property) + @dataclass class SetProperty: """SchemaChange class to set the property and value pairs for the schema.""" _property: str = field(metadata=config(field_name="property")) _value: str = field(metadata=config(field_name="value")) - def __init__(self, schema_property: str, value: str): - self._property = schema_property - self._value = value - def property(self): """Retrieves the name of the property to be set. @@ -95,14 +106,12 @@ def __str__(self): """ return f"SETPROPERTY {self._property} {self._value}" + @dataclass class RemoveProperty: """SchemaChange class to remove a property from the schema.""" _property: str = field(metadata=config(field_name="property")) - def __init__(self, schema_property: str): - self._property = schema_property - def property(self): """Retrieves the name of the property to be removed. diff --git a/clients/client-python/gravitino/api/supports_schemas.py b/clients/client-python/gravitino/api/supports_schemas.py index 12b9cf2a1d2..773201f47ed 100644 --- a/clients/client-python/gravitino/api/supports_schemas.py +++ b/clients/client-python/gravitino/api/supports_schemas.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import ABC, abstractmethod @@ -8,8 +22,6 @@ from gravitino.api.schema import Schema from gravitino.api.schema_change import SchemaChange -from gravitino.name_identifier import NameIdentifier -from gravitino.namespace import Namespace class NoSuchSchemaException(Exception): @@ -25,50 +37,47 @@ class SupportsSchemas(ABC): """ @abstractmethod - def list_schemas(self, namespace: Namespace) -> List[NameIdentifier]: - """List schemas under a namespace. + def list_schemas(self) -> List[str]: + """List schemas under the entity. If an entity such as a table, view exists, its parent schemas must also exist and must be returned by this discovery method. For example, if table a.b.t exists, this method invoked as - list_schemas(a) must return [a.b] in the result array. - - Args: - namespace: The namespace to list. + listSchemas(a) must return [b] in the result array Raises: NoSuchCatalogException: If the catalog does not exist. Returns: - A list of schema identifiers under the namespace. + A list of schema names under the namespace. """ pass - def schema_exists(self, ident: NameIdentifier) -> bool: + def schema_exists(self, schema_name: str) -> bool: """Check if a schema exists. If an entity such as a table, view exists, its parent namespaces must also exist. For example, if table a.b.t exists, this method invoked as schema_exists(a.b) must return true. Args: - ident: The name identifier of the schema. + schema_name: The name of the schema. Returns: True if the schema exists, false otherwise. """ try: - self.load_schema(ident) + self.load_schema(schema_name) return True except NoSuchSchemaException: return False @abstractmethod def create_schema( - self, ident: NameIdentifier, comment: str, properties: Dict[str, str] + self, schema_name: str, comment: str, properties: Dict[str, str] ) -> Schema: """Create a schema in the catalog. Args: - ident: The name identifier of the schema. + schema_name: The name of the schema. comment: The comment of the schema. properties: The properties of the schema. @@ -82,11 +91,11 @@ def create_schema( pass @abstractmethod - def load_schema(self, ident: NameIdentifier) -> Schema: + def load_schema(self, schema_name: str) -> Schema: """Load metadata properties for a schema. Args: - ident: The name identifier of the schema. + schema_name: The name of the schema. Raises: NoSuchSchemaException: If the schema does not exist (optional). @@ -97,11 +106,11 @@ def load_schema(self, ident: NameIdentifier) -> Schema: pass @abstractmethod - def alter_schema(self, ident: NameIdentifier, *changes: SchemaChange) -> Schema: + def alter_schema(self, schema_name: str, *changes: SchemaChange) -> Schema: """Apply the metadata change to a schema in the catalog. Args: - ident: The name identifier of the schema. + schema_name: The name of the schema. changes: The metadata changes to apply. Raises: @@ -113,12 +122,12 @@ def alter_schema(self, ident: NameIdentifier, *changes: SchemaChange) -> Schema: pass @abstractmethod - def drop_schema(self, ident: NameIdentifier, cascade: bool) -> bool: + def drop_schema(self, schema_name: str, cascade: bool) -> bool: """Drop a schema from the catalog. If cascade option is true, recursively drop all objects within the schema. Args: - ident: The name identifier of the schema. + schema_name: The name of the schema. cascade: If true, recursively drop all objects within the schema. Returns: diff --git a/clients/client-python/gravitino/auth/__init__.py b/clients/client-python/gravitino/auth/__init__.py new file mode 100644 index 00000000000..c206137f175 --- /dev/null +++ b/clients/client-python/gravitino/auth/__init__.py @@ -0,0 +1,18 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" diff --git a/clients/client-python/gravitino/auth/auth_constants.py b/clients/client-python/gravitino/auth/auth_constants.py new file mode 100644 index 00000000000..2494030fcdc --- /dev/null +++ b/clients/client-python/gravitino/auth/auth_constants.py @@ -0,0 +1,24 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + + +class AuthConstants: + HTTP_HEADER_AUTHORIZATION: str = "Authorization" + + AUTHORIZATION_BASIC_HEADER: str = "Basic " diff --git a/clients/client-python/gravitino/auth/auth_data_provider.py b/clients/client-python/gravitino/auth/auth_data_provider.py new file mode 100644 index 00000000000..cf28036e991 --- /dev/null +++ b/clients/client-python/gravitino/auth/auth_data_provider.py @@ -0,0 +1,47 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from abc import ABC, abstractmethod + + +class AuthDataProvider(ABC): + """The provider of authentication data""" + + @abstractmethod + def has_token_data(self) -> bool: + """Judge whether AuthDataProvider can provide token data. + + Returns: + true if the AuthDataProvider can provide token data otherwise false. + """ + pass + + @abstractmethod + def get_token_data(self) -> bytes: + """Acquire the data of token for authentication. The client will set the token data as HTTP header + Authorization directly. So the return value should ensure token data contain the token header + (eg: Bearer, Basic) if necessary. + + Returns: + the token data is used for authentication. + """ + + @abstractmethod + def close(self): + """Close the resource in the provider.""" diff --git a/clients/client-python/gravitino/auth/simple_auth_provider.py b/clients/client-python/gravitino/auth/simple_auth_provider.py new file mode 100644 index 00000000000..ef013a7fe4b --- /dev/null +++ b/clients/client-python/gravitino/auth/simple_auth_provider.py @@ -0,0 +1,56 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import base64 +import os + +from .auth_constants import AuthConstants +from .auth_data_provider import AuthDataProvider + + +class SimpleAuthProvider(AuthDataProvider): + """SimpleAuthProvider will use the environment variable `GRAVITINO_USER` or + the user of the system to generate a basic token for every request. + + """ + + _token: bytes + + def __init__(self): + gravitino_user = os.environ.get("GRAVITINO_USER") + if gravitino_user is None or len(gravitino_user) == 0: + gravitino_user = os.environ.get("user.name") + + if gravitino_user is None or len(gravitino_user) == 0: + gravitino_user = "anonymous" + + user_information = f"{gravitino_user}:dummy" + self._token = ( + AuthConstants.AUTHORIZATION_BASIC_HEADER + + base64.b64encode(user_information.encode("utf-8")).decode("utf-8") + ).encode("utf-8") + + def has_token_data(self) -> bool: + return True + + def get_token_data(self) -> bytes: + return self._token + + def close(self): + pass diff --git a/clients/client-python/gravitino/catalog/__init__.py b/clients/client-python/gravitino/catalog/__init__.py index 5779a3ad252..c206137f175 100644 --- a/clients/client-python/gravitino/catalog/__init__.py +++ b/clients/client-python/gravitino/catalog/__init__.py @@ -1,4 +1,18 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ diff --git a/clients/client-python/gravitino/catalog/base_schema_catalog.py b/clients/client-python/gravitino/catalog/base_schema_catalog.py index 2f30d5c7a0e..83d4ea229f8 100644 --- a/clients/client-python/gravitino/catalog/base_schema_catalog.py +++ b/clients/client-python/gravitino/catalog/base_schema_catalog.py @@ -1,10 +1,24 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ import logging -from typing import Dict +from typing import Dict, List from gravitino.api.catalog import Catalog from gravitino.api.schema import Schema @@ -18,9 +32,9 @@ from gravitino.dto.responses.drop_response import DropResponse from gravitino.dto.responses.entity_list_response import EntityListResponse from gravitino.dto.responses.schema_response import SchemaResponse -from gravitino.name_identifier import NameIdentifier from gravitino.namespace import Namespace from gravitino.utils import HTTPClient +from gravitino.rest.rest_utils import encode_string logger = logging.getLogger(__name__) @@ -32,11 +46,15 @@ class BaseSchemaCatalog(CatalogDTO, SupportsSchemas): create, load, alter and drop a schema with specified identifier. """ + # The REST client to send the requests. rest_client: HTTPClient - """The REST client to send the requests.""" + + # The namespace of current catalog, which is the metalake name. + _catalog_namespace: Namespace def __init__( self, + catalog_namespace: Namespace, name: str = None, catalog_type: Catalog.Type = Catalog.Type.UNSUPPORTED, provider: str = None, @@ -54,42 +72,42 @@ def __init__( _audit=audit, ) self.rest_client = rest_client + self._catalog_namespace = catalog_namespace + + self.validate() def as_schemas(self): return self - def list_schemas(self, namespace: Namespace) -> [NameIdentifier]: + def list_schemas(self) -> List[str]: """List all the schemas under the given catalog namespace. - Args: - namespace: The namespace of the catalog. - Raises: NoSuchCatalogException if the catalog with specified namespace does not exist. Returns: - A list of {@link NameIdentifier} of the schemas under the given catalog namespace. + A list of schema names under the given catalog namespace. """ - Namespace.check_schema(namespace) resp = self.rest_client.get( - BaseSchemaCatalog.format_schema_request_path(namespace) + BaseSchemaCatalog.format_schema_request_path(self._schema_namespace()) ) entity_list_response = EntityListResponse.from_json( resp.body, infer_missing=True ) entity_list_response.validate() - return entity_list_response.identifiers() + + return [ident.name() for ident in entity_list_response.identifiers()] def create_schema( self, - ident: NameIdentifier = None, + schema_name: str = None, comment: str = None, properties: Dict[str, str] = None, ) -> Schema: """Create a new schema with specified identifier, comment and metadata. Args: - ident: The name identifier of the schema. + schema_name: The name of the schema. comment: The comment of the schema. properties: The properties of the schema. @@ -100,23 +118,23 @@ def create_schema( Returns: The created Schema. """ - NameIdentifier.check_schema(ident) - req = SchemaCreateRequest(ident.name(), comment, properties) + req = SchemaCreateRequest(encode_string(schema_name), comment, properties) req.validate() resp = self.rest_client.post( - BaseSchemaCatalog.format_schema_request_path(ident.namespace()), json=req + BaseSchemaCatalog.format_schema_request_path(self._schema_namespace()), + json=req, ) schema_response = SchemaResponse.from_json(resp.body, infer_missing=True) schema_response.validate() return schema_response.schema() - def load_schema(self, ident: NameIdentifier) -> Schema: + def load_schema(self, schema_name: str) -> Schema: """Load the schema with specified identifier. Args: - ident: The name identifier of the schema. + schema_name: The name of the schema. Raises: NoSuchSchemaException if the schema with specified identifier does not exist. @@ -124,22 +142,21 @@ def load_schema(self, ident: NameIdentifier) -> Schema: Returns: The Schema with specified identifier. """ - NameIdentifier.check_schema(ident) resp = self.rest_client.get( - BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + BaseSchemaCatalog.format_schema_request_path(self._schema_namespace()) + "/" - + ident.name() + + encode_string(schema_name) ) schema_response = SchemaResponse.from_json(resp.body, infer_missing=True) schema_response.validate() return schema_response.schema() - def alter_schema(self, ident: NameIdentifier, *changes: SchemaChange) -> Schema: + def alter_schema(self, schema_name: str, *changes: SchemaChange) -> Schema: """Alter the schema with specified identifier by applying the changes. Args: - ident: The name identifier of the schema. + schema_name: The name of the schema. changes: The metadata changes to apply. Raises: @@ -148,27 +165,26 @@ def alter_schema(self, ident: NameIdentifier, *changes: SchemaChange) -> Schema: Returns: The altered Schema. """ - NameIdentifier.check_schema(ident) reqs = [ BaseSchemaCatalog.to_schema_update_request(change) for change in changes ] updates_request = SchemaUpdatesRequest(reqs) updates_request.validate() resp = self.rest_client.put( - BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + BaseSchemaCatalog.format_schema_request_path(self._schema_namespace()) + "/" - + ident.name(), + + encode_string(schema_name), updates_request, ) schema_response = SchemaResponse.from_json(resp.body, infer_missing=True) schema_response.validate() return schema_response.schema() - def drop_schema(self, ident: NameIdentifier, cascade: bool) -> bool: + def drop_schema(self, schema_name: str, cascade: bool) -> bool: """Drop the schema with specified identifier. Args: - ident: The name identifier of the schema. + schema_name: The name of the schema. cascade: Whether to drop all the tables under the schema. Raises: @@ -177,22 +193,24 @@ def drop_schema(self, ident: NameIdentifier, cascade: bool) -> bool: Returns: true if the schema is dropped successfully, false otherwise. """ - NameIdentifier.check_schema(ident) try: params = {"cascade": str(cascade)} resp = self.rest_client.delete( - BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + BaseSchemaCatalog.format_schema_request_path(self._schema_namespace()) + "/" - + ident.name(), + + encode_string(schema_name), params=params, ) drop_resp = DropResponse.from_json(resp.body, infer_missing=True) drop_resp.validate() return drop_resp.dropped() except Exception: - logger.warning("Failed to drop schema %s", ident) + logger.warning("Failed to drop schema %s", schema_name) return False + def _schema_namespace(self) -> Namespace: + return Namespace.of(self._catalog_namespace.level(0), self.name()) + @staticmethod def format_schema_request_path(ns: Namespace): return "api/metalakes/" + ns.level(0) + "/catalogs/" + ns.level(1) + "/schemas" @@ -206,3 +224,20 @@ def to_schema_update_request(change: SchemaChange): if isinstance(change, SchemaChange.RemoveProperty): return SchemaUpdateRequest.RemoveSchemaPropertyRequest(change.property()) raise ValueError(f"Unknown change type: {type(change).__name__}") + + def validate(self): + Namespace.check( + self._catalog_namespace is not None + and self._catalog_namespace.length() == 1, + f"Catalog namespace must be non-null and have 1 level, the input namespace is {self._catalog_namespace}", + ) + + assert self.rest_client is not None, "restClient must be set" + assert ( + self.name() is not None and len(self.name().strip()) > 0 + ), "name must not be blank" + assert self.type() is not None, "type must not be None" + assert ( + self.provider() is not None and len(self.provider().strip()) > 0 + ), "provider must not be blank" + assert self.audit_info() is not None, "audit must not be None" diff --git a/clients/client-python/gravitino/catalog/fileset_catalog.py b/clients/client-python/gravitino/catalog/fileset_catalog.py index a7add50dfea..cb38d0c6f8e 100644 --- a/clients/client-python/gravitino/catalog/fileset_catalog.py +++ b/clients/client-python/gravitino/catalog/fileset_catalog.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ import logging @@ -20,6 +34,7 @@ from gravitino.name_identifier import NameIdentifier from gravitino.namespace import Namespace from gravitino.utils import HTTPClient +from gravitino.rest.rest_utils import encode_string logger = logging.getLogger(__name__) @@ -32,6 +47,7 @@ class FilesetCatalog(BaseSchemaCatalog): def __init__( self, + namespace: Namespace, name: str = None, catalog_type: Catalog.Type = Catalog.Type.UNSUPPORTED, provider: str = None, @@ -42,7 +58,14 @@ def __init__( ): super().__init__( - name, catalog_type, provider, comment, properties, audit, rest_client + namespace, + name, + catalog_type, + provider, + comment, + properties, + audit, + rest_client, ) def as_fileset_catalog(self): @@ -52,23 +75,29 @@ def list_filesets(self, namespace: Namespace) -> List[NameIdentifier]: """List the filesets in a schema namespace from the catalog. Args: - namespace A schema namespace. + namespace: A schema namespace. This namespace should have 1 level, which is the schema name Raises: NoSuchSchemaException If the schema does not exist. Returns: - An array of fileset identifiers in the namespace. + A list of NameIdentifier of filesets under the given namespace. """ - Namespace.check_fileset(namespace) - resp = self.rest_client.get(self.format_fileset_request_path(namespace)) + self.check_fileset_namespace(namespace) + + full_namespace = self._get_fileset_full_namespace(namespace) + + resp = self.rest_client.get(self.format_fileset_request_path(full_namespace)) entity_list_resp = EntityListResponse.from_json(resp.body, infer_missing=True) entity_list_resp.validate() - return entity_list_resp.identifiers() + return [ + NameIdentifier.of(ident.namespace().level(2), ident.name()) + for ident in entity_list_resp.identifiers() + ] - def load_fileset(self, ident) -> Fileset: + def load_fileset(self, ident: NameIdentifier) -> Fileset: """Load fileset metadata by {@link NameIdentifier} from the catalog. Args: @@ -80,10 +109,12 @@ def load_fileset(self, ident) -> Fileset: Returns: The fileset metadata. """ - NameIdentifier.check_fileset(ident) + self.check_fileset_name_identifier(ident) + + full_namespace = self._get_fileset_full_namespace(ident.namespace()) resp = self.rest_client.get( - f"{self.format_fileset_request_path(ident.namespace())}/{ident.name()}" + f"{self.format_fileset_request_path(full_namespace)}/{encode_string(ident.name())}" ) fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True) fileset_resp.validate() @@ -119,10 +150,12 @@ def create_fileset( Returns: The created fileset metadata """ - NameIdentifier.check_fileset(ident) + self.check_fileset_name_identifier(ident) + + full_namespace = self._get_fileset_full_namespace(ident.namespace()) req = FilesetCreateRequest( - name=ident.name(), + name=encode_string(ident.name()), comment=comment, fileset_type=fileset_type, storage_location=storage_location, @@ -130,14 +163,14 @@ def create_fileset( ) resp = self.rest_client.post( - self.format_fileset_request_path(ident.namespace()), req + self.format_fileset_request_path(full_namespace), req ) fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True) fileset_resp.validate() return fileset_resp.fileset() - def alter_fileset(self, ident, *changes) -> Fileset: + def alter_fileset(self, ident: NameIdentifier, *changes) -> Fileset: """Update a fileset metadata in the catalog. Args: @@ -151,7 +184,9 @@ def alter_fileset(self, ident, *changes) -> Fileset: Returns: The updated fileset metadata. """ - NameIdentifier.check_fileset(ident) + self.check_fileset_name_identifier(ident) + + full_namespace = self._get_fileset_full_namespace(ident.namespace()) updates = [ FilesetCatalog.to_fileset_update_request(change) for change in changes @@ -160,7 +195,7 @@ def alter_fileset(self, ident, *changes) -> Fileset: req.validate() resp = self.rest_client.put( - f"{self.format_fileset_request_path(ident.namespace())}/{ident.name()}", req + f"{self.format_fileset_request_path(full_namespace)}/{ident.name()}", req ) fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True) fileset_resp.validate() @@ -180,10 +215,12 @@ def drop_fileset(self, ident: NameIdentifier) -> bool: true If the fileset is dropped, false the fileset did not exist. """ try: - NameIdentifier.check_fileset(ident) + self.check_fileset_name_identifier(ident) + + full_namespace = self._get_fileset_full_namespace(ident.namespace()) resp = self.rest_client.delete( - f"{self.format_fileset_request_path(ident.namespace())}/{ident.name()}", + f"{self.format_fileset_request_path(full_namespace)}/{ident.name()}", ) drop_resp = DropResponse.from_json(resp.body, infer_missing=True) drop_resp.validate() @@ -193,10 +230,31 @@ def drop_fileset(self, ident: NameIdentifier) -> bool: logger.warning("Failed to drop fileset %s: %s", ident, e) return False + @staticmethod + def check_fileset_namespace(namespace: Namespace): + Namespace.check( + namespace is not None and namespace.length() == 1, + f"Fileset namespace must be non-null and have 1 level, the input namespace is {namespace}", + ) + + @staticmethod + def check_fileset_name_identifier(ident: NameIdentifier): + NameIdentifier.check(ident is not None, "NameIdentifier must not be None") + NameIdentifier.check( + ident.name() is not None and len(ident.name()) != 0, + "NameIdentifier name must not be empty", + ) + FilesetCatalog.check_fileset_namespace(ident.namespace()) + + def _get_fileset_full_namespace(self, table_namespace: Namespace) -> Namespace: + return Namespace.of( + self._catalog_namespace.level(0), self.name(), table_namespace.level(0) + ) + @staticmethod def format_fileset_request_path(namespace: Namespace) -> str: schema_ns = Namespace.of(namespace.level(0), namespace.level(1)) - return f"{BaseSchemaCatalog.format_schema_request_path(schema_ns)}/{namespace.level(2)}/filesets" + return f"{BaseSchemaCatalog.format_schema_request_path(schema_ns)}/{encode_string(namespace.level(2))}/filesets" @staticmethod def to_fileset_update_request(change: FilesetChange): diff --git a/clients/client-python/gravitino/client/__init__.py b/clients/client-python/gravitino/client/__init__.py index 5779a3ad252..c206137f175 100644 --- a/clients/client-python/gravitino/client/__init__.py +++ b/clients/client-python/gravitino/client/__init__.py @@ -1,4 +1,18 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ diff --git a/clients/client-python/gravitino/client/gravitino_admin_client.py b/clients/client-python/gravitino/client/gravitino_admin_client.py index 2810fd74ead..1bb8ce8e936 100644 --- a/clients/client-python/gravitino/client/gravitino_admin_client.py +++ b/clients/client-python/gravitino/client/gravitino_admin_client.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ import logging @@ -15,7 +29,6 @@ from gravitino.dto.responses.metalake_list_response import MetalakeListResponse from gravitino.dto.responses.metalake_response import MetalakeResponse from gravitino.api.metalake_change import MetalakeChange -from gravitino.name_identifier import NameIdentifier logger = logging.getLogger(__name__) @@ -27,8 +40,6 @@ class GravitinoAdminClient(GravitinoClientBase): Normal users should use {@link GravitinoClient} to connect with the Gravitino server. """ - # TODO: AuthDataProvider authDataProvider - def list_metalakes(self) -> List[GravitinoMetalake]: """Retrieves a list of Metalakes from the Gravitino API. @@ -47,12 +58,12 @@ def list_metalakes(self) -> List[GravitinoMetalake]: ] def create_metalake( - self, ident: NameIdentifier, comment: str, properties: Dict[str, str] + self, name: str, comment: str, properties: Dict[str, str] ) -> GravitinoMetalake: """Creates a new Metalake using the Gravitino API. Args: - ident: The identifier of the new Metalake. + name: The name of the new Metalake. comment: The comment for the new Metalake. properties: The properties of the new Metalake. @@ -60,9 +71,7 @@ def create_metalake( A GravitinoMetalake instance representing the newly created Metalake. TODO: @throws MetalakeAlreadyExistsException If a Metalake with the specified identifier already exists. """ - NameIdentifier.check_metalake(ident) - - req = MetalakeCreateRequest(ident.name(), comment, properties) + req = MetalakeCreateRequest(name, comment, properties) req.validate() resp = self._rest_client.post(self.API_METALAKES_LIST_PATH, req) @@ -72,13 +81,11 @@ def create_metalake( return GravitinoMetalake(metalake, self._rest_client) - def alter_metalake( - self, ident: NameIdentifier, *changes: MetalakeChange - ) -> GravitinoMetalake: + def alter_metalake(self, name: str, *changes: MetalakeChange) -> GravitinoMetalake: """Alters a specific Metalake using the Gravitino API. Args: - ident: The identifier of the Metalake to be altered. + name: The name of the Metalake to be altered. changes: The changes to be applied to the Metalake. Returns: @@ -87,13 +94,12 @@ def alter_metalake( TODO: @throws IllegalArgumentException If the provided changes are invalid or not applicable. """ - NameIdentifier.check_metalake(ident) reqs = [DTOConverters.to_metalake_update_request(change) for change in changes] updates_request = MetalakeUpdatesRequest(reqs) updates_request.validate() resp = self._rest_client.put( - self.API_METALAKES_IDENTIFIER_PATH + ident.name(), updates_request + self.API_METALAKES_IDENTIFIER_PATH + name, updates_request ) metalake_response = MetalakeResponse.from_json(resp.body, infer_missing=True) metalake_response.validate() @@ -101,24 +107,20 @@ def alter_metalake( return GravitinoMetalake(metalake, self._rest_client) - def drop_metalake(self, ident: NameIdentifier) -> bool: + def drop_metalake(self, name: str) -> bool: """Drops a specific Metalake using the Gravitino API. Args: - ident: The identifier of the Metalake to be dropped. + name: The name of the Metalake to be dropped. Returns: True if the Metalake was successfully dropped, false otherwise. """ - NameIdentifier.check_metalake(ident) - try: - resp = self._rest_client.delete( - self.API_METALAKES_IDENTIFIER_PATH + ident.name() - ) + resp = self._rest_client.delete(self.API_METALAKES_IDENTIFIER_PATH + name) drop_response = DropResponse.from_json(resp.body, infer_missing=True) return drop_response.dropped() except Exception: - logger.warning("Failed to drop metalake %s", ident) + logger.warning("Failed to drop metalake %s", name) return False diff --git a/clients/client-python/gravitino/client/gravitino_client.py b/clients/client-python/gravitino/client/gravitino_client.py index 842652125b3..bd2337a1602 100644 --- a/clients/client-python/gravitino/client/gravitino_client.py +++ b/clients/client-python/gravitino/client/gravitino_client.py @@ -1,16 +1,29 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from typing import List, Dict from gravitino.api.catalog import Catalog from gravitino.api.catalog_change import CatalogChange +from gravitino.auth.auth_data_provider import AuthDataProvider from gravitino.client.gravitino_client_base import GravitinoClientBase from gravitino.client.gravitino_metalake import GravitinoMetalake -from gravitino.name_identifier import NameIdentifier -from gravitino.namespace import Namespace class NoSuchMetalakeException(Exception): @@ -26,7 +39,7 @@ class CatalogAlreadyExistsException(Exception): class GravitinoClient(GravitinoClientBase): - """Gravitino Client for an user to interact with the Gravitino API, allowing the client to list, + """Gravitino Client for a user to interact with the Gravitino API, allowing the client to list, load, create, and alter Catalog. It uses an underlying {@link RESTClient} to send HTTP requests and receive responses from the API. @@ -34,19 +47,26 @@ class GravitinoClient(GravitinoClientBase): _metalake: GravitinoMetalake - def __init__(self, uri: str, metalake_name: str): + def __init__( + self, + uri: str, + metalake_name: str, + check_version: bool = True, + auth_data_provider: AuthDataProvider = None, + ): """Constructs a new GravitinoClient with the given URI, authenticator and AuthDataProvider. Args: uri: The base URI for the Gravitino API. metalake_name: The specified metalake name. - TODO: authDataProvider: The provider of the data which is used for authentication. + auth_data_provider: The provider of the data which is used for authentication. Raises: NoSuchMetalakeException if the metalake with specified name does not exist. """ - super().__init__(uri) - self._metalake = super().load_metalake(NameIdentifier.of(metalake_name)) + super().__init__(uri, check_version, auth_data_provider) + self.check_metalake_name(metalake_name) + self._metalake = super().load_metalake(metalake_name) def get_metalake(self) -> GravitinoMetalake: """Get the current metalake object @@ -59,29 +79,29 @@ def get_metalake(self) -> GravitinoMetalake: """ return self._metalake - def list_catalogs(self, namespace: Namespace) -> List[NameIdentifier]: - return self.get_metalake().list_catalogs(namespace) + def list_catalogs(self) -> List[str]: + return self.get_metalake().list_catalogs() - def list_catalogs_info(self, namespace: Namespace) -> List[Catalog]: - return self.get_metalake().list_catalogs_info(namespace) + def list_catalogs_info(self) -> List[Catalog]: + return self.get_metalake().list_catalogs_info() - def load_catalog(self, ident: NameIdentifier) -> Catalog: - return self.get_metalake().load_catalog(ident) + def load_catalog(self, name: str) -> Catalog: + return self.get_metalake().load_catalog(name) def create_catalog( self, - ident: NameIdentifier, + name: str, catalog_type: Catalog.Type, provider: str, comment: str, properties: Dict[str, str], ) -> Catalog: return self.get_metalake().create_catalog( - ident, catalog_type, provider, comment, properties + name, catalog_type, provider, comment, properties ) - def alter_catalog(self, ident: NameIdentifier, *changes: CatalogChange): - return self.get_metalake().alter_catalog(ident, *changes) + def alter_catalog(self, name: str, *changes: CatalogChange): + return self.get_metalake().alter_catalog(name, *changes) - def drop_catalog(self, ident: NameIdentifier): - return self.get_metalake().drop_catalog(ident) + def drop_catalog(self, name: str): + return self.get_metalake().drop_catalog(name) diff --git a/clients/client-python/gravitino/client/gravitino_client_base.py b/clients/client-python/gravitino/client/gravitino_client_base.py index cf17d1e6c39..0bf77071ae8 100644 --- a/clients/client-python/gravitino/client/gravitino_client_base.py +++ b/clients/client-python/gravitino/client/gravitino_client_base.py @@ -1,15 +1,36 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ import logging +import configparser +import os.path +from gravitino.auth.auth_data_provider import AuthDataProvider from gravitino.client.gravitino_metalake import GravitinoMetalake from gravitino.client.gravitino_version import GravitinoVersion +from gravitino.dto.version_dto import VersionDTO from gravitino.dto.responses.metalake_response import MetalakeResponse -from gravitino.name_identifier import NameIdentifier +from gravitino.dto.responses.version_response import VersionResponse from gravitino.utils import HTTPClient +from gravitino.exceptions.gravitino_runtime_exception import GravitinoRuntimeException +from gravitino.constants.version import VERSION_INI, Version +from gravitino.name_identifier import NameIdentifier logger = logging.getLogger(__name__) @@ -29,14 +50,21 @@ class GravitinoClientBase: API_METALAKES_IDENTIFIER_PATH = f"{API_METALAKES_LIST_PATH}/" """The REST API path prefix for load a specific metalake""" - def __init__(self, uri: str): - self._rest_client = HTTPClient(uri) - - def load_metalake(self, ident: NameIdentifier) -> GravitinoMetalake: + def __init__( + self, + uri: str, + check_version: bool = True, + auth_data_provider: AuthDataProvider = None, + ): + self._rest_client = HTTPClient(uri, auth_data_provider=auth_data_provider) + if check_version: + self.check_version() + + def load_metalake(self, name: str) -> GravitinoMetalake: """Loads a specific Metalake from the Gravitino API. Args: - ident The identifier of the Metalake to be loaded. + name: The name of the Metalake to be loaded. Returns: A GravitinoMetalake instance representing the loaded Metalake. @@ -45,10 +73,9 @@ def load_metalake(self, ident: NameIdentifier) -> GravitinoMetalake: NoSuchMetalakeException If the specified Metalake does not exist. """ - NameIdentifier.check_metalake(ident) - + self.check_metalake_name(name) response = self._rest_client.get( - GravitinoClientBase.API_METALAKES_IDENTIFIER_PATH + ident.name() + GravitinoClientBase.API_METALAKES_IDENTIFIER_PATH + name ) metalake_response = MetalakeResponse.from_json( response.body, infer_missing=True @@ -57,16 +84,53 @@ def load_metalake(self, ident: NameIdentifier) -> GravitinoMetalake: return GravitinoMetalake(metalake_response.metalake(), self._rest_client) - def get_version(self) -> GravitinoVersion: + def check_version(self): + """Check the compatibility of the client with the target server. + + Raises: + GravitinoRuntimeException If the client version is greater than the server version. + """ + server_version = self.get_server_version() + client_version = self.get_client_version() + + if client_version > server_version: + raise GravitinoRuntimeException( + "Gravitino does not support the case that " + "the client-side version is higher than the server-side version." + f"The client version is {client_version.version()}, and the server version {server_version.version()}" + ) + + def get_client_version(self) -> GravitinoVersion: + """Retrieves the version of the Gravitino Python Client. + + Returns: + A GravitinoVersion instance representing the version of the Gravitino Python Client. + """ + config = configparser.ConfigParser() + + if not os.path.exists(VERSION_INI): + raise GravitinoRuntimeException( + f"Failed to get Gravitino version, version file '{VERSION_INI}' does not exist." + ) + config.read(VERSION_INI) + + version = config["metadata"][Version.VERSION.value] + compile_date = config["metadata"][Version.COMPILE_DATE.value] + git_commit = config["metadata"][Version.GIT_COMMIT.value] + + return GravitinoVersion(VersionDTO(version, compile_date, git_commit)) + + def get_server_version(self) -> GravitinoVersion: """Retrieves the version of the Gravitino API. Returns: A GravitinoVersion instance representing the version of the Gravitino API. """ resp = self._rest_client.get("api/version") - resp.validate() + version_response = VersionResponse.from_json(resp.body, infer_missing=True) + version_response.validate() - return GravitinoVersion(resp.get_version()) + return GravitinoVersion(version_response.version()) def close(self): """Closes the GravitinoClient and releases any underlying resources.""" @@ -75,3 +139,12 @@ def close(self): self._rest_client.close() except Exception as e: logger.warning("Failed to close the HTTP REST client: %s", e) + + def check_metalake_name(self, metalake_name: str): + identifier = NameIdentifier.parse(metalake_name) + namespace = identifier.namespace() + + if not namespace: + raise ValueError( + f"Metalake namespace must be empty, the input namespace is {namespace}" + ) diff --git a/clients/client-python/gravitino/client/gravitino_metalake.py b/clients/client-python/gravitino/client/gravitino_metalake.py index 4b774681de0..e4009fc17d9 100644 --- a/clients/client-python/gravitino/client/gravitino_metalake.py +++ b/clients/client-python/gravitino/client/gravitino_metalake.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ import logging @@ -16,8 +30,6 @@ from gravitino.dto.responses.catalog_response import CatalogResponse from gravitino.dto.responses.drop_response import DropResponse from gravitino.dto.responses.entity_list_response import EntityListResponse -from gravitino.name_identifier import NameIdentifier -from gravitino.namespace import Namespace from gravitino.utils import HTTPClient @@ -56,30 +68,23 @@ def __init__(self, metalake: MetalakeDTO = None, client: HTTPClient = None): ) self.rest_client = client - def list_catalogs(self, namespace: Namespace) -> List[NameIdentifier]: - """List all the catalogs under this metalake with specified namespace. - - Args: - namespace The namespace to list the catalogs under it. + def list_catalogs(self) -> List[str]: + """List all the catalogs under this metalake. Raises: NoSuchMetalakeException if the metalake with specified namespace does not exist. Returns: - A list of {@link NameIdentifier} of the catalogs under the specified namespace. + A list of the catalog names under this metalake. """ - Namespace.check_catalog(namespace) - url = f"api/metalakes/{namespace.level(0)}/catalogs" + url = f"api/metalakes/{self.name()}/catalogs" response = self.rest_client.get(url) entity_list = EntityListResponse.from_json(response.body, infer_missing=True) entity_list.validate() - return entity_list.identifiers() + return [identifier.name() for identifier in entity_list.identifiers()] - def list_catalogs_info(self, namespace: Namespace) -> List[Catalog]: - """List all the catalogs with their information under this metalake with specified namespace. - - Args: - namespace The namespace to list the catalogs under it. + def list_catalogs_info(self) -> List[Catalog]: + """List all the catalogs with their information under this metalake. Raises: NoSuchMetalakeException if the metalake with specified namespace does not exist. @@ -87,66 +92,63 @@ def list_catalogs_info(self, namespace: Namespace) -> List[Catalog]: Returns: A list of Catalog under the specified namespace. """ - Namespace.check_catalog(namespace) params = {"details": "true"} - url = f"api/metalakes/{namespace.level(0)}/catalogs" + url = f"api/metalakes/{self.name()}/catalogs" response = self.rest_client.get(url, params=params) catalog_list = CatalogListResponse.from_json(response.body, infer_missing=True) return [ - DTOConverters.to_catalog(catalog, self.rest_client) + DTOConverters.to_catalog(self.name(), catalog, self.rest_client) for catalog in catalog_list.catalogs() ] - def load_catalog(self, ident: NameIdentifier) -> Catalog: - """Load the catalog with specified identifier. + def load_catalog(self, name: str) -> Catalog: + """Load the catalog with specified name. Args: - ident: The identifier of the catalog to load. + name: The name of the catalog to load. Raises: - NoSuchCatalogException if the catalog with specified identifier does not exist. + NoSuchCatalogException if the catalog with specified name does not exist. Returns: - The Catalog with specified identifier. + The Catalog with specified name. """ - NameIdentifier.check_catalog(ident) - url = self.API_METALAKES_CATALOGS_PATH.format( - ident.namespace().level(0), ident.name() - ) + url = self.API_METALAKES_CATALOGS_PATH.format(self.name(), name) response = self.rest_client.get(url) catalog_resp = CatalogResponse.from_json(response.body, infer_missing=True) - return DTOConverters.to_catalog(catalog_resp.catalog(), self.rest_client) + return DTOConverters.to_catalog( + self.name(), catalog_resp.catalog(), self.rest_client + ) def create_catalog( self, - ident: NameIdentifier, + name: str, catalog_type: Catalog.Type, provider: str, comment: str, properties: Dict[str, str], ) -> Catalog: - """Create a new catalog with specified identifier, catalog type, comment and properties. + """Create a new catalog with specified name, catalog type, comment and properties. Args: - ident: The identifier of the catalog. + name: The name of the catalog. catalog_type: The type of the catalog. provider: The provider of the catalog. comment: The comment of the catalog. properties: The properties of the catalog. Raises: - NoSuchMetalakeException if the metalake with specified namespace does not exist. - CatalogAlreadyExistsException if the catalog with specified identifier already exists. + NoSuchMetalakeException if the metalake does not exist. + CatalogAlreadyExistsException if the catalog with specified name already exists. Returns: The created Catalog. """ - NameIdentifier.check_catalog(ident) catalog_create_request = CatalogCreateRequest( - name=ident.name(), + name=name, catalog_type=catalog_type, provider=provider, comment=comment, @@ -154,54 +156,53 @@ def create_catalog( ) catalog_create_request.validate() - url = f"api/metalakes/{ident.namespace().level(0)}/catalogs" + url = f"api/metalakes/{self.name()}/catalogs" response = self.rest_client.post(url, json=catalog_create_request) catalog_resp = CatalogResponse.from_json(response.body, infer_missing=True) - return DTOConverters.to_catalog(catalog_resp.catalog(), self.rest_client) + return DTOConverters.to_catalog( + self.name(), catalog_resp.catalog(), self.rest_client + ) - def alter_catalog(self, ident: NameIdentifier, *changes: CatalogChange) -> Catalog: - """Alter the catalog with specified identifier by applying the changes. + def alter_catalog(self, name: str, *changes: CatalogChange) -> Catalog: + """Alter the catalog with specified name by applying the changes. Args: - ident: the identifier of the catalog. + name: the name of the catalog. changes: the changes to apply to the catalog. Raises: - NoSuchCatalogException if the catalog with specified identifier does not exist. + NoSuchCatalogException if the catalog with specified name does not exist. IllegalArgumentException if the changes are invalid. Returns: the altered Catalog. """ - NameIdentifier.check_catalog(ident) reqs = [DTOConverters.to_catalog_update_request(change) for change in changes] updates_request = CatalogUpdatesRequest(reqs) updates_request.validate() - url = self.API_METALAKES_CATALOGS_PATH.format( - ident.namespace().level(0), ident.name() - ) + url = self.API_METALAKES_CATALOGS_PATH.format(self.name(), name) response = self.rest_client.put(url, json=updates_request) catalog_response = CatalogResponse.from_json(response.body, infer_missing=True) catalog_response.validate() - return DTOConverters.to_catalog(catalog_response.catalog(), self.rest_client) + return DTOConverters.to_catalog( + self.name(), catalog_response.catalog(), self.rest_client + ) - def drop_catalog(self, ident: NameIdentifier) -> bool: - """Drop the catalog with specified identifier. + def drop_catalog(self, name: str) -> bool: + """Drop the catalog with specified name. Args: - ident the identifier of the catalog. + name the name of the catalog. Returns: true if the catalog is dropped successfully, false otherwise. """ try: - url = self.API_METALAKES_CATALOGS_PATH.format( - ident.namespace().level(0), ident.name() - ) + url = self.API_METALAKES_CATALOGS_PATH.format(self.name(), name) response = self.rest_client.delete(url) drop_response = DropResponse.from_json(response.body, infer_missing=True) @@ -209,5 +210,5 @@ def drop_catalog(self, ident: NameIdentifier) -> bool: return drop_response.dropped() except Exception: - logger.warning("Failed to drop catalog %s", ident) + logger.warning("Failed to drop catalog %s", name) return False diff --git a/clients/client-python/gravitino/client/gravitino_version.py b/clients/client-python/gravitino/client/gravitino_version.py index f3f004bbf60..f59e78a86ee 100644 --- a/clients/client-python/gravitino/client/gravitino_version.py +++ b/clients/client-python/gravitino/client/gravitino_version.py @@ -1,18 +1,84 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ +import re from dataclasses import dataclass +from enum import Enum from gravitino.dto.version_dto import VersionDTO +from gravitino.exceptions.gravitino_runtime_exception import GravitinoRuntimeException + + +class Version(Enum): + MAJOR = "MAJOR" + MINOR = "MINOR" + PATCH = "PATCH" + + +VERSION_PATTERN: str = ( + rf"(?P<{Version.MAJOR.value}>\d+)\.(?P<{Version.MINOR.value}>\d+)\.(?P<{Version.PATCH.value}>\d+)+?" +) @dataclass class GravitinoVersion(VersionDTO): """Gravitino version information.""" + major: int + minor: int + patch: int + def __init__(self, versionDTO): super().__init__( - versionDTO.version, versionDTO.compile_date, versionDTO.git_commit + versionDTO.version(), versionDTO.compile_date(), versionDTO.git_commit() ) + + m = re.match(VERSION_PATTERN, self.version()) + + assert m is not None, "Invalid version string " + self.version() + + self.major = int(m.group(Version.MAJOR.value)) + self.minor = int(m.group(Version.MINOR.value)) + self.patch = int(m.group(Version.PATCH.value)) + + def __gt__(self, other) -> bool: + if not isinstance(other, GravitinoVersion): + raise GravitinoRuntimeException( + f"{GravitinoVersion.__name__} can't compare with {other.__class__.__name__}" + ) + if self.major > other.major: + return True + if self.minor > other.minor: + return True + if self.patch > other.patch: + return True + return False + + def __eq__(self, other) -> bool: + if not isinstance(other, GravitinoVersion): + raise GravitinoRuntimeException( + f"{GravitinoVersion.__name__} can't compare with {other.__class__.__name__}" + ) + if self.major != other.major: + return False + if self.minor != other.minor: + return False + if self.patch != other.patch: + return False + return True diff --git a/clients/client-python/gravitino/constants.py b/clients/client-python/gravitino/constants.py deleted file mode 100644 index bca1d178c49..00000000000 --- a/clients/client-python/gravitino/constants.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -TIMEOUT = 10 diff --git a/clients/client-python/gravitino/constants/__init__.py b/clients/client-python/gravitino/constants/__init__.py new file mode 100644 index 00000000000..c206137f175 --- /dev/null +++ b/clients/client-python/gravitino/constants/__init__.py @@ -0,0 +1,18 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" diff --git a/clients/client-python/gravitino/constants/doc.py b/clients/client-python/gravitino/constants/doc.py new file mode 100644 index 00000000000..86df9809fe2 --- /dev/null +++ b/clients/client-python/gravitino/constants/doc.py @@ -0,0 +1,22 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from gravitino.constants.root import PROJECT_HOME + +DOC_DIR = PROJECT_HOME / "docs" diff --git a/clients/client-python/gravitino/constants/root.py b/clients/client-python/gravitino/constants/root.py new file mode 100644 index 00000000000..613983f1f3b --- /dev/null +++ b/clients/client-python/gravitino/constants/root.py @@ -0,0 +1,24 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from pathlib import Path + +MODULE_NAME = "gravitino" +PROJECT_HOME = Path(__file__).parent.parent.parent +GRAVITINO_DIR = PROJECT_HOME / MODULE_NAME diff --git a/clients/client-python/gravitino/constants/timeout.py b/clients/client-python/gravitino/constants/timeout.py new file mode 100644 index 00000000000..5977e00ce33 --- /dev/null +++ b/clients/client-python/gravitino/constants/timeout.py @@ -0,0 +1,20 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +TIMEOUT = 10 diff --git a/clients/client-python/gravitino/constants/version.py b/clients/client-python/gravitino/constants/version.py new file mode 100644 index 00000000000..ffcfeea55c9 --- /dev/null +++ b/clients/client-python/gravitino/constants/version.py @@ -0,0 +1,32 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from enum import Enum + +from gravitino.constants.root import PROJECT_HOME +from gravitino.constants.root import MODULE_NAME + +VERSION_INI = PROJECT_HOME / MODULE_NAME / "version.ini" +SETUP_FILE = PROJECT_HOME / "setup.py" + + +class Version(Enum): + VERSION = "version" + GIT_COMMIT = "gitCommit" + COMPILE_DATE = "compileDate" diff --git a/clients/client-python/gravitino/dto/__init__.py b/clients/client-python/gravitino/dto/__init__.py index 5779a3ad252..c206137f175 100644 --- a/clients/client-python/gravitino/dto/__init__.py +++ b/clients/client-python/gravitino/dto/__init__.py @@ -1,4 +1,18 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ diff --git a/clients/client-python/gravitino/dto/audit_dto.py b/clients/client-python/gravitino/dto/audit_dto.py index 6ce56ca892e..e8f1253a60c 100644 --- a/clients/client-python/gravitino/dto/audit_dto.py +++ b/clients/client-python/gravitino/dto/audit_dto.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/catalog_dto.py b/clients/client-python/gravitino/dto/catalog_dto.py index 3a483a5bef8..4e48c4ef688 100644 --- a/clients/client-python/gravitino/dto/catalog_dto.py +++ b/clients/client-python/gravitino/dto/catalog_dto.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from typing import Dict diff --git a/clients/client-python/gravitino/dto/dto_converters.py b/clients/client-python/gravitino/dto/dto_converters.py index a131db954ab..86e614557a6 100644 --- a/clients/client-python/gravitino/dto/dto_converters.py +++ b/clients/client-python/gravitino/dto/dto_converters.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from gravitino.api.catalog import Catalog @@ -11,6 +25,7 @@ from gravitino.dto.requests.metalake_update_request import MetalakeUpdateRequest from gravitino.api.metalake_change import MetalakeChange from gravitino.utils import HTTPClient +from gravitino.namespace import Namespace class DTOConverters: @@ -37,9 +52,11 @@ def to_metalake_update_request(change: MetalakeChange) -> object: raise ValueError(f"Unknown change type: {type(change).__name__}") @staticmethod - def to_catalog(catalog: CatalogDTO, client: HTTPClient): + def to_catalog(metalake: str, catalog: CatalogDTO, client: HTTPClient): + namespace = Namespace.of(metalake) if catalog.type() == Catalog.Type.FILESET: return FilesetCatalog( + namespace=namespace, name=catalog.name(), catalog_type=catalog.type(), provider=catalog.provider(), @@ -54,9 +71,11 @@ def to_catalog(catalog: CatalogDTO, client: HTTPClient): @staticmethod def to_catalog_update_request(change: CatalogChange): if isinstance(change, CatalogChange.RenameCatalog): - return CatalogUpdateRequest.RenameCatalogRequest(change.new_name) + return CatalogUpdateRequest.RenameCatalogRequest(change.new_name()) if isinstance(change, CatalogChange.UpdateCatalogComment): - return CatalogUpdateRequest.UpdateCatalogCommentRequest(change.new_comment) + return CatalogUpdateRequest.UpdateCatalogCommentRequest( + change.new_comment() + ) if isinstance(change, CatalogChange.SetProperty): # TODO # pylint: disable=too-many-function-args @@ -64,6 +83,6 @@ def to_catalog_update_request(change: CatalogChange): change.property(), change.value() ) if isinstance(change, CatalogChange.RemoveProperty): - return CatalogUpdateRequest.RemoveCatalogPropertyRequest(change._property) + return CatalogUpdateRequest.RemoveCatalogPropertyRequest(change.property()) raise ValueError(f"Unknown change type: {type(change).__name__}") diff --git a/clients/client-python/gravitino/dto/fileset_dto.py b/clients/client-python/gravitino/dto/fileset_dto.py index 2ea41495221..120e6713ac6 100644 --- a/clients/client-python/gravitino/dto/fileset_dto.py +++ b/clients/client-python/gravitino/dto/fileset_dto.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/metalake_dto.py b/clients/client-python/gravitino/dto/metalake_dto.py index dfdb8573ae2..38730ff8512 100644 --- a/clients/client-python/gravitino/dto/metalake_dto.py +++ b/clients/client-python/gravitino/dto/metalake_dto.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field @@ -49,10 +63,10 @@ def equals(self, other): if not isinstance(other, MetalakeDTO): return False return ( - self._name == other._name - and self._comment == other._comment - and self.property_equal(self._properties, other._properties) - and self._audit == other._audit + self.name() == other.name() + and self.comment() == other.comment() + and self.property_equal(self.properties(), other.properties()) + and self.audit_info() == other.audit_info() ) def property_equal(self, p1, p2): diff --git a/clients/client-python/gravitino/dto/requests/__init__.py b/clients/client-python/gravitino/dto/requests/__init__.py index 5779a3ad252..c206137f175 100644 --- a/clients/client-python/gravitino/dto/requests/__init__.py +++ b/clients/client-python/gravitino/dto/requests/__init__.py @@ -1,4 +1,18 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ diff --git a/clients/client-python/gravitino/dto/requests/catalog_create_request.py b/clients/client-python/gravitino/dto/requests/catalog_create_request.py index 9dc806563e8..c8f70dd11df 100644 --- a/clients/client-python/gravitino/dto/requests/catalog_create_request.py +++ b/clients/client-python/gravitino/dto/requests/catalog_create_request.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/requests/catalog_update_request.py b/clients/client-python/gravitino/dto/requests/catalog_update_request.py index c8dc720015a..d3ba6298afc 100644 --- a/clients/client-python/gravitino/dto/requests/catalog_update_request.py +++ b/clients/client-python/gravitino/dto/requests/catalog_update_request.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import abstractmethod @@ -28,11 +42,19 @@ def catalog_change(self): class CatalogUpdateRequest: """Represents an interface for catalog update requests.""" + @dataclass class RenameCatalogRequest(CatalogUpdateRequestBase): - new_name: Optional[str] = field(metadata=config(field_name="newName")) + """Represents a request to rename a catalog.""" + + _new_name: Optional[str] = field(metadata=config(field_name="newName")) + """The new name for the catalog.""" + + def __init__(self, new_name: str): + super().__init__("rename") + self._new_name = new_name def catalog_change(self): - return CatalogChange.rename(self.new_name) + return CatalogChange.rename(self._new_name) def validate(self): """Validates the fields of the request. @@ -41,46 +63,66 @@ def validate(self): IllegalArgumentException if the new name is not set. """ assert ( - self.new_name is None + self._new_name is None ), '"newName" field is required and cannot be empty' + @dataclass class UpdateCatalogCommentRequest(CatalogUpdateRequestBase): """Request to update the comment of a catalog.""" - new_comment: Optional[str] = field(metadata=config(field_name="newComment")) + _new_comment: Optional[str] = field(metadata=config(field_name="newComment")) + """The new comment for the catalog.""" + + def __init__(self, new_comment: str): + super().__init__("updateComment") + self._new_comment = new_comment def catalog_change(self): - return CatalogChange.update_comment(self.new_comment) + return CatalogChange.update_comment(self._new_comment) def validate(self): assert ( - self.new_comment is None + self._new_comment is None ), '"newComment" field is required and cannot be empty' + @dataclass class SetCatalogPropertyRequest(CatalogUpdateRequestBase): """Request to set a property on a catalog.""" - property: Optional[str] = None - value: Optional[str] = None + _property: Optional[str] = field(metadata=config(field_name="property")) + """The property to set.""" + + _value: Optional[str] = field(metadata=config(field_name="value")) + """The value of the property.""" + + def __init__(self, catalog_property: str, value: str): + super().__init__("setProperty") + self._property = catalog_property + self._value = value def catalog_change(self): - return CatalogChange.set_property(self.property, self.value) + return CatalogChange.set_property(self._property, self._value) def validate(self): assert ( - self.property is None + self._property is None ), '"property" field is required and cannot be empty' - assert self.value is None, '"value" field is required and cannot be empty' + assert self._value is None, '"value" field is required and cannot be empty' class RemoveCatalogPropertyRequest(CatalogUpdateRequestBase): """Request to remove a property from a catalog.""" property: Optional[str] = None + """The property to remove.""" + + def __init__(self, catalog_property: str): + super().__init__("removeProperty") + self._property = catalog_property def catalog_change(self): - return CatalogChange.remove_property(self.property) + return CatalogChange.remove_property(self._property) def validate(self): assert ( - self.property is None + self._property is None ), '"property" field is required and cannot be empty' diff --git a/clients/client-python/gravitino/dto/requests/catalog_updates_request.py b/clients/client-python/gravitino/dto/requests/catalog_updates_request.py index 8437aa070a2..942611445b4 100644 --- a/clients/client-python/gravitino/dto/requests/catalog_updates_request.py +++ b/clients/client-python/gravitino/dto/requests/catalog_updates_request.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/requests/fileset_create_request.py b/clients/client-python/gravitino/dto/requests/fileset_create_request.py index 2c4d0e76873..86d69326610 100644 --- a/clients/client-python/gravitino/dto/requests/fileset_create_request.py +++ b/clients/client-python/gravitino/dto/requests/fileset_create_request.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/requests/fileset_update_request.py b/clients/client-python/gravitino/dto/requests/fileset_update_request.py index 37805d89b4f..319207bb868 100644 --- a/clients/client-python/gravitino/dto/requests/fileset_update_request.py +++ b/clients/client-python/gravitino/dto/requests/fileset_update_request.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import abstractmethod diff --git a/clients/client-python/gravitino/dto/requests/fileset_updates_request.py b/clients/client-python/gravitino/dto/requests/fileset_updates_request.py index 4f3c02b86b7..f5a45da0849 100644 --- a/clients/client-python/gravitino/dto/requests/fileset_updates_request.py +++ b/clients/client-python/gravitino/dto/requests/fileset_updates_request.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/requests/metalake_create_request.py b/clients/client-python/gravitino/dto/requests/metalake_create_request.py index e1d49e3b8c1..2308e965bc5 100644 --- a/clients/client-python/gravitino/dto/requests/metalake_create_request.py +++ b/clients/client-python/gravitino/dto/requests/metalake_create_request.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from typing import Optional, Dict diff --git a/clients/client-python/gravitino/dto/requests/metalake_update_request.py b/clients/client-python/gravitino/dto/requests/metalake_update_request.py index 02f2d2aefb2..bcb9c910194 100644 --- a/clients/client-python/gravitino/dto/requests/metalake_update_request.py +++ b/clients/client-python/gravitino/dto/requests/metalake_update_request.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import abstractmethod diff --git a/clients/client-python/gravitino/dto/requests/metalake_updates_request.py b/clients/client-python/gravitino/dto/requests/metalake_updates_request.py index 016e1eeaf66..d5ecec69cb5 100644 --- a/clients/client-python/gravitino/dto/requests/metalake_updates_request.py +++ b/clients/client-python/gravitino/dto/requests/metalake_updates_request.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/requests/schema_create_request.py b/clients/client-python/gravitino/dto/requests/schema_create_request.py index 0d3154d2975..5e09e2f2bc1 100644 --- a/clients/client-python/gravitino/dto/requests/schema_create_request.py +++ b/clients/client-python/gravitino/dto/requests/schema_create_request.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/requests/schema_update_request.py b/clients/client-python/gravitino/dto/requests/schema_update_request.py index 288e54b50b6..7ce8b653cce 100644 --- a/clients/client-python/gravitino/dto/requests/schema_update_request.py +++ b/clients/client-python/gravitino/dto/requests/schema_update_request.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import abstractmethod diff --git a/clients/client-python/gravitino/dto/requests/schema_updates_request.py b/clients/client-python/gravitino/dto/requests/schema_updates_request.py index 05d59c2d4fe..29a5ef9e7bb 100644 --- a/clients/client-python/gravitino/dto/requests/schema_updates_request.py +++ b/clients/client-python/gravitino/dto/requests/schema_updates_request.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/responses/__init__.py b/clients/client-python/gravitino/dto/responses/__init__.py index 5779a3ad252..c206137f175 100644 --- a/clients/client-python/gravitino/dto/responses/__init__.py +++ b/clients/client-python/gravitino/dto/responses/__init__.py @@ -1,4 +1,18 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ diff --git a/clients/client-python/gravitino/dto/responses/base_response.py b/clients/client-python/gravitino/dto/responses/base_response.py index e272c95928e..71efece86f6 100644 --- a/clients/client-python/gravitino/dto/responses/base_response.py +++ b/clients/client-python/gravitino/dto/responses/base_response.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/responses/catalog_list_response.py b/clients/client-python/gravitino/dto/responses/catalog_list_response.py index cb8b2f2580f..6feca21dcd1 100644 --- a/clients/client-python/gravitino/dto/responses/catalog_list_response.py +++ b/clients/client-python/gravitino/dto/responses/catalog_list_response.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/responses/catalog_response.py b/clients/client-python/gravitino/dto/responses/catalog_response.py index 6aba8d06bc2..7118165638d 100644 --- a/clients/client-python/gravitino/dto/responses/catalog_response.py +++ b/clients/client-python/gravitino/dto/responses/catalog_response.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/responses/drop_response.py b/clients/client-python/gravitino/dto/responses/drop_response.py index 8515135a600..9ac58554f47 100644 --- a/clients/client-python/gravitino/dto/responses/drop_response.py +++ b/clients/client-python/gravitino/dto/responses/drop_response.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/responses/entity_list_response.py b/clients/client-python/gravitino/dto/responses/entity_list_response.py index dc3cd71486e..4e32ac4e48e 100644 --- a/clients/client-python/gravitino/dto/responses/entity_list_response.py +++ b/clients/client-python/gravitino/dto/responses/entity_list_response.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/responses/fileset_response.py b/clients/client-python/gravitino/dto/responses/fileset_response.py index 56a5fbec62f..db0a1d88609 100644 --- a/clients/client-python/gravitino/dto/responses/fileset_response.py +++ b/clients/client-python/gravitino/dto/responses/fileset_response.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/responses/metalake_list_response.py b/clients/client-python/gravitino/dto/responses/metalake_list_response.py index 182b2d96d4b..925c4532ba1 100644 --- a/clients/client-python/gravitino/dto/responses/metalake_list_response.py +++ b/clients/client-python/gravitino/dto/responses/metalake_list_response.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/responses/metalake_response.py b/clients/client-python/gravitino/dto/responses/metalake_response.py index e7d36231596..c3c4be1033b 100644 --- a/clients/client-python/gravitino/dto/responses/metalake_response.py +++ b/clients/client-python/gravitino/dto/responses/metalake_response.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/responses/schema_response.py b/clients/client-python/gravitino/dto/responses/schema_response.py index 1a630eea6e4..836b5376e80 100644 --- a/clients/client-python/gravitino/dto/responses/schema_response.py +++ b/clients/client-python/gravitino/dto/responses/schema_response.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/responses/version_response.py b/clients/client-python/gravitino/dto/responses/version_response.py new file mode 100644 index 00000000000..6994dbca155 --- /dev/null +++ b/clients/client-python/gravitino/dto/responses/version_response.py @@ -0,0 +1,53 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from dataclasses import dataclass, field +from dataclasses_json import config + +from .base_response import BaseResponse +from ..version_dto import VersionDTO + + +@dataclass +class VersionResponse(BaseResponse): + """Represents a response containing version of Gravitino.""" + + _version: VersionDTO = field(metadata=config(field_name="version")) + + def version(self) -> VersionDTO: + return self._version + + def validate(self): + """Validates the response data. + + Raise: + IllegalArgumentException if name or audit information is not set. + """ + super().validate() + + assert self._version is not None, "version must be non-null" + assert ( + self._version.version() is not None + ), "version 'version' must not be null and empty" + assert ( + self._version.compile_date() is not None + ), "version 'compile_date' must not be null and empty" + assert ( + self._version.git_commit() is not None + ), "version 'git_commit' must not be null and empty" diff --git a/clients/client-python/gravitino/dto/schema_dto.py b/clients/client-python/gravitino/dto/schema_dto.py index 12739a60183..d0ba90d8681 100644 --- a/clients/client-python/gravitino/dto/schema_dto.py +++ b/clients/client-python/gravitino/dto/schema_dto.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from dataclasses import dataclass, field diff --git a/clients/client-python/gravitino/dto/version_dto.py b/clients/client-python/gravitino/dto/version_dto.py index 5a56b638af3..304b8f1a382 100644 --- a/clients/client-python/gravitino/dto/version_dto.py +++ b/clients/client-python/gravitino/dto/version_dto.py @@ -1,20 +1,44 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ -from dataclasses import dataclass +from dataclasses import dataclass, field +from dataclasses_json import config @dataclass class VersionDTO: """Represents a Version Data Transfer Object (DTO).""" - version: str = "" + _version: str = field(metadata=config(field_name="version")) """The version of the software.""" - compile_date: str = "" + _compile_date: str = field(metadata=config(field_name="compileDate")) """The date the software was compiled.""" - git_commit: str = "" + _git_commit: str = field(metadata=config(field_name="gitCommit")) """The git commit of the software.""" + + def version(self) -> str: + return self._version + + def compile_date(self) -> str: + return self._compile_date + + def git_commit(self) -> str: + return self._git_commit diff --git a/clients/client-python/gravitino/exceptions/__init__.py b/clients/client-python/gravitino/exceptions/__init__.py index 5779a3ad252..c206137f175 100644 --- a/clients/client-python/gravitino/exceptions/__init__.py +++ b/clients/client-python/gravitino/exceptions/__init__.py @@ -1,4 +1,18 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ diff --git a/clients/client-python/gravitino/exceptions/gravitino_runtime_exception.py b/clients/client-python/gravitino/exceptions/gravitino_runtime_exception.py index 520b6a7ec5a..d98bc169c80 100644 --- a/clients/client-python/gravitino/exceptions/gravitino_runtime_exception.py +++ b/clients/client-python/gravitino/exceptions/gravitino_runtime_exception.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ @@ -8,4 +22,4 @@ class GravitinoRuntimeException(RuntimeError): """Base class for all Gravitino runtime exceptions.""" def __init__(self, message, *args): - super().__init__(message.format(*args)) + super().__init__(message % args) diff --git a/clients/client-python/gravitino/exceptions/illegal_name_identifier_exception.py b/clients/client-python/gravitino/exceptions/illegal_name_identifier_exception.py index f8b5970ba1a..bb14af12c8b 100644 --- a/clients/client-python/gravitino/exceptions/illegal_name_identifier_exception.py +++ b/clients/client-python/gravitino/exceptions/illegal_name_identifier_exception.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ diff --git a/clients/client-python/gravitino/exceptions/illegal_namespace_exception.py b/clients/client-python/gravitino/exceptions/illegal_namespace_exception.py index eecfdcfb5c5..f8ded707047 100644 --- a/clients/client-python/gravitino/exceptions/illegal_namespace_exception.py +++ b/clients/client-python/gravitino/exceptions/illegal_namespace_exception.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ diff --git a/clients/client-python/gravitino/exceptions/no_such_metalake_exception.py b/clients/client-python/gravitino/exceptions/no_such_metalake_exception.py index 942e81c99a5..6dc136ca92b 100644 --- a/clients/client-python/gravitino/exceptions/no_such_metalake_exception.py +++ b/clients/client-python/gravitino/exceptions/no_such_metalake_exception.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from gravitino.exceptions.not_found_exception import NotFoundException diff --git a/clients/client-python/gravitino/exceptions/not_found_exception.py b/clients/client-python/gravitino/exceptions/not_found_exception.py index 7c8ca0ca51b..aff8cb583b1 100644 --- a/clients/client-python/gravitino/exceptions/not_found_exception.py +++ b/clients/client-python/gravitino/exceptions/not_found_exception.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from gravitino.exceptions.gravitino_runtime_exception import GravitinoRuntimeException diff --git a/clients/client-python/gravitino/filesystem/__init__.py b/clients/client-python/gravitino/filesystem/__init__.py new file mode 100644 index 00000000000..c206137f175 --- /dev/null +++ b/clients/client-python/gravitino/filesystem/__init__.py @@ -0,0 +1,18 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py new file mode 100644 index 00000000000..edb150677d6 --- /dev/null +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -0,0 +1,689 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from enum import Enum +from pathlib import PurePosixPath +from typing import Dict, Tuple +import re +import fsspec + +from cachetools import TTLCache +from fsspec import AbstractFileSystem +from fsspec.implementations.local import LocalFileSystem +from fsspec.implementations.arrow import ArrowFSWrapper +from fsspec.utils import infer_storage_options +from pyarrow.fs import HadoopFileSystem +from readerwriterlock import rwlock +from gravitino.api.catalog import Catalog +from gravitino.api.fileset import Fileset +from gravitino.client.gravitino_client import GravitinoClient +from gravitino.exceptions.gravitino_runtime_exception import GravitinoRuntimeException +from gravitino.name_identifier import NameIdentifier + +PROTOCOL_NAME = "gvfs" + + +class StorageType(Enum): + HDFS = "hdfs" + LOCAL = "file" + + +class FilesetContext: + """A context object that holds the information about the fileset and the file system which used in + the GravitinoVirtualFileSystem's operations. + """ + + def __init__( + self, + name_identifier: NameIdentifier, + fileset: Fileset, + fs: AbstractFileSystem, + storage_type: StorageType, + actual_path: str, + ): + self._name_identifier = name_identifier + self._fileset = fileset + self._fs = fs + self._storage_type = storage_type + self._actual_path = actual_path + + def get_name_identifier(self): + return self._name_identifier + + def get_fileset(self): + return self._fileset + + def get_fs(self): + return self._fs + + def get_actual_path(self): + return self._actual_path + + def get_storage_type(self): + return self._storage_type + + +class GravitinoVirtualFileSystem(fsspec.AbstractFileSystem): + """This is a virtual file system which users can access `fileset` and + other resources. + + It obtains the actual storage location corresponding to the resource from the + Gravitino server, and creates an independent file system for it to act as an agent for users to + access the underlying storage. + """ + + # Override the parent variable + protocol = PROTOCOL_NAME + _identifier_pattern = re.compile("^fileset/([^/]+)/([^/]+)/([^/]+)(?:/[^/]+)*/?$") + + def __init__( + self, + server_uri=None, + metalake_name=None, + cache_size=20, + cache_expired_time=3600, + **kwargs, + ): + self._metalake = metalake_name + self._client = GravitinoClient( + uri=server_uri, metalake_name=metalake_name, check_version=False + ) + self._cache = TTLCache(maxsize=cache_size, ttl=cache_expired_time) + self._cache_lock = rwlock.RWLockFair() + + super().__init__(**kwargs) + + @property + def cache(self): + return self._cache + + @property + def fsid(self): + return PROTOCOL_NAME + + def sign(self, path, expiration=None, **kwargs): + """We do not support to create a signed URL representing the given path in gvfs.""" + raise GravitinoRuntimeException( + "Sign is not implemented for Gravitino Virtual FileSystem." + ) + + def ls(self, path, detail=True, **kwargs): + """List the files and directories info of the path. + :param path: Virtual fileset path + :param detail: Whether to show the details for the files and directories info + :param kwargs: Extra args + :return If details is true, returns a list of file info dicts, else returns a list of file paths + """ + context: FilesetContext = self._get_fileset_context(path) + if detail: + entries = [ + self._convert_actual_info(entry, context) + for entry in context.get_fs().ls( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ), + detail=True, + ) + ] + return entries + entries = [ + self._convert_actual_path(entry_path, context) + for entry_path in context.get_fs().ls( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ), + detail=False, + ) + ] + return entries + + def info(self, path, **kwargs): + """Get file info. + :param path: Virtual fileset path + :param kwargs: Extra args + :return A file info dict + """ + context: FilesetContext = self._get_fileset_context(path) + actual_info: Dict = context.get_fs().info( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ) + ) + return self._convert_actual_info(actual_info, context) + + def exists(self, path, **kwargs): + """Check if a file or a directory exists. + :param path: Virtual fileset path + :param kwargs: Extra args + :return If a file or directory exists, it returns True, otherwise False + """ + context: FilesetContext = self._get_fileset_context(path) + return context.get_fs().exists( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ) + ) + + def cp_file(self, path1, path2, **kwargs): + """Copy a file. + :param path1: Virtual src fileset path + :param path2: Virtual dst fileset path, should be consistent with the src path fileset identifier + :param kwargs: Extra args + """ + src_path = self._pre_process_path(path1) + dst_path = self._pre_process_path(path2) + src_identifier: NameIdentifier = self._extract_identifier(src_path) + dst_identifier: NameIdentifier = self._extract_identifier(dst_path) + if src_identifier != dst_identifier: + raise GravitinoRuntimeException( + f"Destination file path identifier: `{dst_identifier}` should be same with src file path " + f"identifier: `{src_identifier}`." + ) + src_context: FilesetContext = self._get_fileset_context(src_path) + if self._check_mount_single_file( + src_context.get_fileset(), + src_context.get_fs(), + src_context.get_storage_type(), + ): + raise GravitinoRuntimeException( + f"Cannot cp file of the fileset: {src_identifier} which only mounts to a single file." + ) + dst_context: FilesetContext = self._get_fileset_context(dst_path) + + src_context.get_fs().cp_file( + self._strip_storage_protocol( + src_context.get_storage_type(), src_context.get_actual_path() + ), + self._strip_storage_protocol( + dst_context.get_storage_type(), dst_context.get_actual_path() + ), + ) + + def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs): + """Move a file to another directory. + This can move a file to another existing directory. + If the target path directory does not exist, an exception will be thrown. + :param path1: Virtual src fileset path + :param path2: Virtual dst fileset path, should be consistent with the src path fileset identifier + :param recursive: Whether to move recursively + :param maxdepth: Maximum depth of recursive move + :param kwargs: Extra args + """ + src_path = self._pre_process_path(path1) + dst_path = self._pre_process_path(path2) + src_identifier: NameIdentifier = self._extract_identifier(src_path) + dst_identifier: NameIdentifier = self._extract_identifier(dst_path) + if src_identifier != dst_identifier: + raise GravitinoRuntimeException( + f"Destination file path identifier: `{dst_identifier}`" + f" should be same with src file path identifier: `{src_identifier}`." + ) + src_context: FilesetContext = self._get_fileset_context(src_path) + if self._check_mount_single_file( + src_context.get_fileset(), + src_context.get_fs(), + src_context.get_storage_type(), + ): + raise GravitinoRuntimeException( + f"Cannot cp file of the fileset: {src_identifier} which only mounts to a single file." + ) + dst_context: FilesetContext = self._get_fileset_context(dst_path) + if src_context.get_storage_type() == StorageType.HDFS: + src_context.get_fs().mv( + self._strip_storage_protocol( + src_context.get_storage_type(), src_context.get_actual_path() + ), + self._strip_storage_protocol( + dst_context.get_storage_type(), dst_context.get_actual_path() + ), + ) + elif src_context.get_storage_type() == StorageType.LOCAL: + src_context.get_fs().mv( + self._strip_storage_protocol( + src_context.get_storage_type(), src_context.get_actual_path() + ), + self._strip_storage_protocol( + dst_context.get_storage_type(), dst_context.get_actual_path() + ), + recursive, + maxdepth, + ) + else: + raise GravitinoRuntimeException( + f"Storage type:{src_context.get_storage_type()} doesn't support now." + ) + + def _rm(self, path): + raise GravitinoRuntimeException( + "Deprecated method, use `rm_file` method instead." + ) + + def rm(self, path, recursive=False, maxdepth=None): + """Remove a file or directory. + :param path: Virtual fileset path + :param recursive: Whether to remove the directory recursively. + When removing a directory, this parameter should be True. + :param maxdepth: The maximum depth to remove the directory recursively. + """ + context: FilesetContext = self._get_fileset_context(path) + context.get_fs().rm( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ), + recursive, + maxdepth, + ) + + def rm_file(self, path): + """Remove a file. + :param path: Virtual fileset path + """ + context: FilesetContext = self._get_fileset_context(path) + context.get_fs().rm_file( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ) + ) + + def rmdir(self, path): + """Remove a directory. + It will delete a directory and all its contents recursively for PyArrow.HadoopFileSystem. + And it will throw an exception if delete a directory which is non-empty for LocalFileSystem. + :param path: Virtual fileset path + """ + context: FilesetContext = self._get_fileset_context(path) + context.get_fs().rmdir( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ) + ) + + def open( + self, + path, + mode="rb", + block_size=None, + cache_options=None, + compression=None, + **kwargs, + ): + """Open a file to read/write/append. + :param path: Virtual fileset path + :param mode: The mode now supports: rb(read), wb(write), ab(append). See builtin ``open()`` + :param block_size: Some indication of buffering - this is a value in bytes + :param cache_options: Extra arguments to pass through to the cache + :param compression: If given, open file using compression codec + :param kwargs: Extra args + :return A file-like object from the filesystem + """ + context: FilesetContext = self._get_fileset_context(path) + return context.get_fs().open( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ), + mode, + block_size, + cache_options, + compression, + **kwargs, + ) + + def mkdir(self, path, create_parents=True, **kwargs): + """Make a directory. + if create_parents=True, this is equivalent to ``makedirs``. + + :param path: Virtual fileset path + :param create_parents: Create parent directories if missing when set to True + :param kwargs: Extra args + """ + context: FilesetContext = self._get_fileset_context(path) + context.get_fs().mkdir( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ), + create_parents, + **kwargs, + ) + + def makedirs(self, path, exist_ok=True): + """Make a directory recursively. + :param path: Virtual fileset path + :param exist_ok: Continue if a directory already exists + """ + context: FilesetContext = self._get_fileset_context(path) + context.get_fs().makedirs( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ), + exist_ok, + ) + + def created(self, path): + """Return the created timestamp of a file as a datetime.datetime + Only supports for `fsspec.LocalFileSystem` now. + :param path: Virtual fileset path + :return Created time(datetime.datetime) + """ + context: FilesetContext = self._get_fileset_context(path) + if context.get_storage_type() == StorageType.LOCAL: + return context.get_fs().created( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ) + ) + raise GravitinoRuntimeException( + f"Storage type:{context.get_storage_type()} doesn't support now." + ) + + def modified(self, path): + """Returns the modified time of the path file if it exists. + :param path: Virtual fileset path + :return Modified time(datetime.datetime) + """ + context: FilesetContext = self._get_fileset_context(path) + return context.get_fs().modified( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ) + ) + + def cat_file(self, path, start=None, end=None, **kwargs): + """Get the content of a file. + :param path: Virtual fileset path + :param start: The offset in bytes to start reading from. It can be None. + :param end: The offset in bytes to end reading at. It can be None. + :param kwargs: Extra args + :return File content + """ + context: FilesetContext = self._get_fileset_context(path) + return context.get_fs().cat_file( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ), + start, + end, + **kwargs, + ) + + def get_file(self, rpath, lpath, callback=None, outfile=None, **kwargs): + """Copy single remote file to local. + :param rpath: Remote file path + :param lpath: Local file path + :param callback: The callback class + :param outfile: The output file path + :param kwargs: Extra args + """ + if not lpath.startswith(f"{StorageType.LOCAL.value}:") and not lpath.startswith( + "/" + ): + raise GravitinoRuntimeException( + "Doesn't support copy a remote gvfs file to an another remote file." + ) + context: FilesetContext = self._get_fileset_context(rpath) + context.get_fs().get_file( + self._strip_storage_protocol( + context.get_storage_type(), context.get_actual_path() + ), + lpath, + **kwargs, + ) + + def _convert_actual_path(self, path, context: FilesetContext): + """Convert an actual path to a virtual path. + The virtual path is like `fileset/{catalog}/{schema}/{fileset}/xxx`. + :param path: Actual path + :param context: Fileset context + :return A virtual path + """ + if context.get_storage_type() == StorageType.HDFS: + actual_prefix = infer_storage_options( + context.get_fileset().storage_location() + )["path"] + elif context.get_storage_type() == StorageType.LOCAL: + actual_prefix = context.get_fileset().storage_location()[ + len(f"{StorageType.LOCAL.value}:") : + ] + else: + raise GravitinoRuntimeException( + f"Storage type:{context.get_storage_type()} doesn't support now." + ) + + if not path.startswith(actual_prefix): + raise GravitinoRuntimeException( + f"Path {path} does not start with valid prefix {actual_prefix}." + ) + virtual_location = self._get_virtual_location(context.get_name_identifier()) + return f"{path.replace(actual_prefix, virtual_location)}" + + def _convert_actual_info(self, entry: Dict, context: FilesetContext): + """Convert a file info from an actual entry to a virtual entry. + :param entry: A dict of the actual file info + :param context: Fileset context + :return A dict of the virtual file info + """ + path = self._convert_actual_path(entry["name"], context) + return { + "name": path, + "size": entry["size"], + "type": entry["type"], + "mtime": entry["mtime"], + } + + def _get_fileset_context(self, virtual_path: str): + """Get a fileset context from the cache or the Gravitino server + :param virtual_path: The virtual path + :return A fileset context + """ + virtual_path: str = self._pre_process_path(virtual_path) + identifier: NameIdentifier = self._extract_identifier(virtual_path) + read_lock = self._cache_lock.gen_rlock() + try: + read_lock.acquire() + cache_value: Tuple[Fileset, AbstractFileSystem, StorageType] = ( + self._cache.get(identifier) + ) + if cache_value is not None: + actual_path = self._get_actual_path_by_ident( + identifier, + cache_value[0], + cache_value[1], + cache_value[2], + virtual_path, + ) + return FilesetContext( + identifier, + cache_value[0], + cache_value[1], + cache_value[2], + actual_path, + ) + finally: + read_lock.release() + + write_lock = self._cache_lock.gen_wlock() + try: + write_lock.acquire() + cache_value: Tuple[Fileset, AbstractFileSystem] = self._cache.get( + identifier + ) + if cache_value is not None: + actual_path = self._get_actual_path_by_ident( + identifier, + cache_value[0], + cache_value[1], + cache_value[2], + virtual_path, + ) + return FilesetContext( + identifier, + cache_value[0], + cache_value[1], + cache_value[2], + actual_path, + ) + fileset: Fileset = self._load_fileset_from_server(identifier) + storage_location = fileset.storage_location() + if storage_location.startswith(f"{StorageType.HDFS.value}://"): + fs = ArrowFSWrapper(HadoopFileSystem.from_uri(storage_location)) + storage_type = StorageType.HDFS + elif storage_location.startswith(f"{StorageType.LOCAL.value}:/"): + fs = LocalFileSystem() + storage_type = StorageType.LOCAL + else: + raise GravitinoRuntimeException( + f"Storage under the fileset: `{identifier}` doesn't support now." + ) + actual_path = self._get_actual_path_by_ident( + identifier, fileset, fs, storage_type, virtual_path + ) + self._cache[identifier] = (fileset, fs, storage_type) + context = FilesetContext(identifier, fileset, fs, storage_type, actual_path) + return context + finally: + write_lock.release() + + def _extract_identifier(self, path): + """Extract the fileset identifier from the path. + :param path: The virtual fileset path + :return The fileset identifier + """ + if path is None: + raise GravitinoRuntimeException( + "path which need be extracted cannot be null or empty." + ) + + match = self._identifier_pattern.match(path) + if match and len(match.groups()) == 3: + return NameIdentifier.of( + self._metalake, match.group(1), match.group(2), match.group(3) + ) + raise GravitinoRuntimeException( + f"path: `{path}` doesn't contains valid identifier." + ) + + def _load_fileset_from_server(self, identifier: NameIdentifier) -> Fileset: + """Load the fileset from the server. + If the fileset is not found on the server, an `NoSuchFilesetException` exception will be raised. + :param identifier: The fileset identifier + :return The fileset + """ + catalog: Catalog = self._client.load_catalog(identifier.namespace().level(1)) + + return catalog.as_fileset_catalog().load_fileset( + NameIdentifier.of(identifier.namespace().level(2), identifier.name()) + ) + + def _get_actual_path_by_ident( + self, + identifier: NameIdentifier, + fileset: Fileset, + fs: AbstractFileSystem, + storage_type: StorageType, + virtual_path: str, + ): + """Get the actual path by the virtual path and the fileset. + :param identifier: The fileset identifier + :param fileset: The fileset + :param fs: The file system corresponding to the fileset storage location + :param storage_type: The storage type of the fileset storage location + :param virtual_path: The virtual fileset path + :return The actual path. + """ + virtual_location = self._get_virtual_location(identifier) + storage_location = fileset.storage_location() + if self._check_mount_single_file(fileset, fs, storage_type): + if virtual_path != virtual_location: + raise GravitinoRuntimeException( + f"Path: {virtual_path} should be same with the virtual location: {virtual_location}" + " when the fileset only mounts a single file." + ) + return storage_location + return virtual_path.replace(virtual_location, storage_location, 1) + + @staticmethod + def _get_virtual_location(identifier: NameIdentifier): + """Get the virtual location of the fileset. + :param identifier: The name identifier of the fileset + :return The virtual location. + """ + return ( + f"fileset/{identifier.namespace().level(1)}" + f"/{identifier.namespace().level(2)}" + f"/{identifier.name()}" + ) + + def _check_mount_single_file( + self, fileset: Fileset, fs: AbstractFileSystem, storage_type: StorageType + ): + """Check if the fileset is mounted a single file. + :param fileset: The fileset + :param fs: The file system corresponding to the fileset storage location + :param storage_type: The storage type of the fileset storage location + :return True the fileset is mounted a single file. + """ + result: Dict = fs.info( + self._strip_storage_protocol(storage_type, fileset.storage_location()) + ) + return result["type"] == "file" + + @staticmethod + def _pre_process_path(virtual_path): + """Pre-process the path. + We will uniformly process `gvfs://fileset/{catalog}/{schema}/{fileset_name}/xxx` + into the format of `fileset/{catalog}/{schema}/{fileset_name}/xxx`. + This is because some implementations of `PyArrow` and `fsspec` can only recognize this format. + :param virtual_path: The virtual path + :return The pre-processed path + """ + if isinstance(virtual_path, PurePosixPath): + pre_processed_path = virtual_path.as_posix() + else: + pre_processed_path = virtual_path + gvfs_prefix = f"{PROTOCOL_NAME}://" + if pre_processed_path.startswith(gvfs_prefix): + pre_processed_path = pre_processed_path[len(gvfs_prefix) :] + if not pre_processed_path.startswith("fileset/"): + raise GravitinoRuntimeException( + f"Invalid path:`{pre_processed_path}`. Expected path to start with `fileset/`." + " Example: fileset/{fileset_catalog}/{schema}/{fileset_name}/{sub_path}." + ) + return pre_processed_path + + @staticmethod + def _strip_storage_protocol(storage_type: StorageType, path: str): + """Strip the storage protocol from the path. + Before passing the path to the underlying file system for processing, + pre-process the protocol information in the path. + Some file systems require special processing. + For HDFS, we can pass the path like 'hdfs://{host}:{port}/xxx'. + For Local, we can pass the path like '/tmp/xxx'. + :param storage_type: The storage type + :param path: The path + :return: The stripped path + """ + if storage_type == StorageType.HDFS: + return path + if storage_type == StorageType.LOCAL: + return path[len(f"{StorageType.LOCAL.value}:") :] + raise GravitinoRuntimeException( + f"Storage type:{storage_type} doesn't support now." + ) + + +fsspec.register_implementation(PROTOCOL_NAME, GravitinoVirtualFileSystem) diff --git a/clients/client-python/gravitino/name_identifier.py b/clients/client-python/gravitino/name_identifier.py index f894a859402..069e225002a 100644 --- a/clients/client-python/gravitino/name_identifier.py +++ b/clients/client-python/gravitino/name_identifier.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from typing import ClassVar @@ -14,6 +28,8 @@ ) from gravitino.namespace import Namespace +# TODO: delete redundant methods + @dataclass class NameIdentifier(DataClassJsonMixin): @@ -26,7 +42,13 @@ class NameIdentifier(DataClassJsonMixin): _DOT: ClassVar[str] = "." _name: str = field(metadata=config(field_name="name")) - _namespace: Namespace = field(metadata=config(field_name="namespace")) + _namespace: Namespace = field( + metadata=config( + field_name="namespace", + encoder=Namespace.to_json, + decoder=Namespace.from_json, + ) + ) @classmethod def builder(cls, namespace: Namespace, name: str): @@ -58,176 +80,6 @@ def of(*names: str) -> "NameIdentifier": return NameIdentifier.builder(Namespace.of(*names[:-1]), names[-1]) - @staticmethod - def of_namespace(namespace: Namespace, name: str) -> "NameIdentifier": - """Create the NameIdentifier with the given Namespace and name. - - Args: - namespace: The namespace of the identifier - name: The name of the identifier - - Returns: - The created NameIdentifier - """ - return NameIdentifier.builder(namespace, name) - - @staticmethod - def of_metalake(metalake: str) -> "NameIdentifier": - """Create the metalake NameIdentifier with the given name. - - Args: - metalake: The metalake name - - Returns: - The created metalake NameIdentifier - """ - return NameIdentifier.of(metalake) - - @staticmethod - def of_catalog(metalake: str, catalog: str) -> "NameIdentifier": - """Create the catalog NameIdentifier with the given metalake and catalog name. - - Args: - metalake: The metalake name - catalog: The catalog name - - Returns: - The created catalog NameIdentifier - """ - return NameIdentifier.of(metalake, catalog) - - @staticmethod - def of_schema(metalake: str, catalog: str, schema: str) -> "NameIdentifier": - """Create the schema NameIdentifier with the given metalake, catalog and schema name. - - Args: - metalake: The metalake name - catalog: The catalog name - schema: The schema name - - Returns: - The created schema NameIdentifier - """ - return NameIdentifier.of(metalake, catalog, schema) - - @staticmethod - def of_table( - metalake: str, catalog: str, schema: str, table: str - ) -> "NameIdentifier": - """Create the table NameIdentifier with the given metalake, catalog, schema and table name. - - Args: - metalake: The metalake name - catalog: The catalog name - schema: The schema name - table: The table name - - Returns: - The created table NameIdentifier - """ - return NameIdentifier.of(metalake, catalog, schema, table) - - @staticmethod - def of_fileset( - metalake: str, catalog: str, schema: str, fileset: str - ) -> "NameIdentifier": - """Create the fileset NameIdentifier with the given metalake, catalog, schema and fileset name. - - Args: - metalake: The metalake name - catalog: The catalog name - schema: The schema name - fileset: The fileset name - - Returns: - The created fileset NameIdentifier - """ - return NameIdentifier.of(metalake, catalog, schema, fileset) - - @staticmethod - def of_topic( - metalake: str, catalog: str, schema: str, topic: str - ) -> "NameIdentifier": - """Create the topic NameIdentifier with the given metalake, catalog, schema and topic - name. - - Args: - metalake: The metalake name - catalog: The catalog name - schema: The schema name - topic: The topic name - - Returns: - The created topic NameIdentifier - """ - return NameIdentifier.of(metalake, catalog, schema, topic) - - @staticmethod - def check_metalake(ident: "NameIdentifier") -> None: - """Check the given NameIdentifier is a metalake identifier. Throw an {@link - IllegalNameIdentifierException} if it's not. - - Args: - ident: The metalake NameIdentifier to check. - """ - NameIdentifier.check(ident is not None, "Metalake identifier must not be null") - Namespace.check_metalake(ident.namespace()) - - @staticmethod - def check_catalog(ident: "NameIdentifier") -> None: - """Check the given NameIdentifier is a catalog identifier. Throw an {@link - IllegalNameIdentifierException} if it's not. - - Args: - ident: The catalog NameIdentifier to check. - """ - NameIdentifier.check(ident is not None, "Catalog identifier must not be null") - Namespace.check_catalog(ident.namespace()) - - @staticmethod - def check_schema(ident: "NameIdentifier") -> None: - """Check the given NameIdentifier is a schema identifier. Throw an {@link - IllegalNameIdentifierException} if it's not. - - Args: - ident: The schema NameIdentifier to check. - """ - NameIdentifier.check(ident is not None, "Schema identifier must not be null") - Namespace.check_schema(ident.namespace()) - - @staticmethod - def check_table(ident: "NameIdentifier") -> None: - """Check the given NameIdentifier is a table identifier. Throw an {@link - IllegalNameIdentifierException} if it's not. - - Args: - ident: The table NameIdentifier to check. - """ - NameIdentifier.check(ident is not None, "Table identifier must not be null") - Namespace.check_table(ident.namespace()) - - @staticmethod - def check_fileset(ident: "NameIdentifier") -> None: - """Check the given NameIdentifier is a fileset identifier. Throw an {@link - IllegalNameIdentifierException} if it's not. - - Args: - ident: The fileset NameIdentifier to check. - """ - NameIdentifier.check(ident is not None, "Fileset identifier must not be null") - Namespace.check_fileset(ident.namespace()) - - @staticmethod - def check_topic(ident: "NameIdentifier") -> None: - """Check the given NameIdentifier is a topic identifier. Throw an {@link - IllegalNameIdentifierException} if it's not. - - Args: - ident: The topic NameIdentifier to check. - """ - NameIdentifier.check(ident is not None, "Topic identifier must not be null") - Namespace.check_topic(ident.namespace()) - @staticmethod def parse(identifier: str) -> "NameIdentifier": """Create a NameIdentifier from the given identifier string. diff --git a/clients/client-python/gravitino/namespace.py b/clients/client-python/gravitino/namespace.py index 69bb85172ba..33747ed2508 100644 --- a/clients/client-python/gravitino/namespace.py +++ b/clients/client-python/gravitino/namespace.py @@ -1,10 +1,27 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ +import json from typing import List, ClassVar +# TODO: delete redundant methods + class Namespace: """A namespace is a sequence of levels separated by dots. It's used to identify a metalake, a @@ -14,11 +31,21 @@ class Namespace: _DOT: ClassVar[str] = "." - _levels: List[str] = [] + _levels: List[str] def __init__(self, levels: List[str]): self._levels = levels + def to_json(self): + return json.dumps(self._levels) + + @classmethod + def from_json(cls, levels): + assert levels is not None and isinstance( + levels, list + ), f"Cannot parse name identifier from invalid JSON: {levels}" + return cls(levels) + @staticmethod def empty() -> "Namespace": """Get an empty namespace. @@ -52,160 +79,6 @@ def of(*levels: str) -> "Namespace": return Namespace(list(levels)) - @staticmethod - def of_metalake() -> "Namespace": - """Create a namespace for metalake. - - Returns: - A namespace for metalake - """ - return Namespace.empty() - - @staticmethod - def of_catalog(metalake: str) -> "Namespace": - """Create a namespace for catalog. - - Args: - metalake: The metalake name - - Returns: - A namespace for catalog - """ - return Namespace.of(metalake) - - @staticmethod - def of_schema(metalake: str, catalog: str) -> "Namespace": - """Create a namespace for schema. - - Args: - metalake: The metalake name - catalog: The catalog name - - Returns: - A namespace for schema - """ - return Namespace.of(metalake, catalog) - - @staticmethod - def of_table(metalake: str, catalog: str, schema: str) -> "Namespace": - """Create a namespace for table. - - Args: - metalake: The metalake name - catalog: The catalog name - schema: The schema name - - Returns: - A namespace for table - """ - return Namespace.of(metalake, catalog, schema) - - @staticmethod - def of_fileset(metalake: str, catalog: str, schema: str) -> "Namespace": - """Create a namespace for fileset. - - Args: - metalake: The metalake name - catalog: The catalog name - schema: The schema name - - Returns: - A namespace for fileset - """ - return Namespace.of(metalake, catalog, schema) - - @staticmethod - def of_topic(metalake: str, catalog: str, schema: str) -> "Namespace": - """Create a namespace for topic. - - Args: - metalake: The metalake name - catalog: The catalog name - schema: The schema name - - Returns: - A namespace for topic - """ - return Namespace.of(metalake, catalog, schema) - - @staticmethod - def check_metalake(namespace: "Namespace") -> None: - """Check if the given metalake namespace is legal, throw an IllegalNamespaceException if - it's illegal. - - Args: - namespace: The metalake namespace - """ - Namespace.check( - namespace is not None and namespace.is_empty(), - f"Metalake namespace must be non-null and empty, the input namespace is {namespace}", - ) - - @staticmethod - def check_catalog(namespace: "Namespace") -> None: - """Check if the given catalog namespace is legal, throw an IllegalNamespaceException if - it's illegal. - - Args: - namespace: The catalog namespace - """ - Namespace.check( - namespace is not None and namespace.length() == 1, - f"Catalog namespace must be non-null and have 1 level, the input namespace is {namespace}", - ) - - @staticmethod - def check_schema(namespace: "Namespace") -> None: - """Check if the given schema namespace is legal, throw an IllegalNamespaceException if - it's illegal. - - Args: - namespace: The schema namespace - """ - Namespace.check( - namespace is not None and namespace.length() == 2, - f"Schema namespace must be non-null and have 2 levels, the input namespace is {namespace}", - ) - - @staticmethod - def check_table(namespace: "Namespace") -> None: - """Check if the given table namespace is legal, throw an IllegalNamespaceException if it's - illegal. - - Args: - namespace: The table namespace - """ - Namespace.check( - namespace is not None and namespace.length() == 3, - f"Table namespace must be non-null and have 3 levels, the input namespace is {namespace}", - ) - - @staticmethod - def check_fileset(namespace: "Namespace") -> None: - """Check if the given fileset namespace is legal, throw an IllegalNamespaceException if - it's illegal. - - Args: - namespace: The fileset namespace - """ - Namespace.check( - namespace is not None and namespace.length() == 3, - f"Fileset namespace must be non-null and have 3 levels, the input namespace is {namespace}", - ) - - @staticmethod - def check_topic(namespace: "Namespace") -> None: - """Check if the given topic namespace is legal, throw an IllegalNamespaceException if it's - illegal. - - Args: - namespace: The topic namespace - """ - Namespace.check( - namespace is not None and namespace.length() == 3, - f"Topic namespace must be non-null and have 3 levels, the input namespace is {namespace}", - ) - def levels(self) -> List[str]: """Get the levels of the namespace. diff --git a/clients/client-python/gravitino/rest/__init__.py b/clients/client-python/gravitino/rest/__init__.py index 5779a3ad252..c206137f175 100644 --- a/clients/client-python/gravitino/rest/__init__.py +++ b/clients/client-python/gravitino/rest/__init__.py @@ -1,4 +1,18 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ diff --git a/clients/client-python/gravitino/rest/rest_message.py b/clients/client-python/gravitino/rest/rest_message.py index 92b30eac18d..59e93325b66 100644 --- a/clients/client-python/gravitino/rest/rest_message.py +++ b/clients/client-python/gravitino/rest/rest_message.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from abc import ABC, abstractmethod diff --git a/clients/client-python/gravitino/rest/rest_utils.py b/clients/client-python/gravitino/rest/rest_utils.py new file mode 100644 index 00000000000..077511a2f52 --- /dev/null +++ b/clients/client-python/gravitino/rest/rest_utils.py @@ -0,0 +1,27 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import urllib.parse + + +def encode_string(to_encode: str): + + assert to_encode is not None, "Invalid string to encode: None" + + return urllib.parse.quote(to_encode) diff --git a/clients/client-python/gravitino/typing.py b/clients/client-python/gravitino/typing.py index 49112533a53..0191bc65225 100644 --- a/clients/client-python/gravitino/typing.py +++ b/clients/client-python/gravitino/typing.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from typing import Mapping, Sequence, Union diff --git a/clients/client-python/gravitino/utils/__init__.py b/clients/client-python/gravitino/utils/__init__.py index 03295d89182..f1e7becb3f4 100644 --- a/clients/client-python/gravitino/utils/__init__.py +++ b/clients/client-python/gravitino/utils/__init__.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ from gravitino.utils.http_client import Response, HTTPClient, unpack diff --git a/clients/client-python/gravitino/utils/http_client.py b/clients/client-python/gravitino/utils/http_client.py index a781171a5b9..150c00002f0 100644 --- a/clients/client-python/gravitino/utils/http_client.py +++ b/clients/client-python/gravitino/utils/http_client.py @@ -28,9 +28,11 @@ from urllib.error import HTTPError import json as _json +from gravitino.auth.auth_constants import AuthConstants +from gravitino.auth.auth_data_provider import AuthDataProvider from gravitino.typing import JSONType from gravitino.utils.exceptions import handle_error -from gravitino.constants import TIMEOUT +from gravitino.constants.timeout import TIMEOUT logger = logging.getLogger(__name__) @@ -66,6 +68,7 @@ def headers(self): def json(self): if self.body: return _json.loads(self.body.decode("utf-8")) + return None class HTTPClient: @@ -76,11 +79,13 @@ def __init__( request_headers=None, timeout=TIMEOUT, is_debug=False, + auth_data_provider: AuthDataProvider = None, ) -> None: self.host = host self.request_headers = request_headers or {} self.timeout = timeout self.is_debug = is_debug + self.auth_data_provider = auth_data_provider def _build_url(self, endpoint=None, params=None): url = self.host @@ -143,7 +148,11 @@ def _request( request.add_header(key, value) if request_data and ("Content-Type" not in self.request_headers): request.add_header("Content-Type", "application/json") - + if self.auth_data_provider is not None: + request.add_header( + AuthConstants.HTTP_HEADER_AUTHORIZATION, + self.auth_data_provider.get_token_data().decode("utf-8"), + ) request.get_method = lambda: method return Response(self._make_request(opener, request, timeout=timeout)) @@ -161,6 +170,8 @@ def put(self, endpoint, json=None, **kwargs): def close(self): self._request("close", "/") + if self.auth_data_provider is not None: + self.auth_data_provider.close() def unpack(path: str): diff --git a/clients/client-python/pylintrc b/clients/client-python/pylintrc index d78d0ebd59d..f866b90afac 100644 --- a/clients/client-python/pylintrc +++ b/clients/client-python/pylintrc @@ -1,5 +1,20 @@ -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [MESSAGES CONTROL] @@ -25,12 +40,7 @@ disable=missing-class-docstring, missing-timeout, #TODO-fix broad-exception-caught, #TODO-fix duplicate-code, #TODO-fix - method-hidden, #TODO-fix - no-name-in-module, #TODO-fix - invalid-field-call, #TODO-fix - protected-access, #TODO-fix too-many-arguments, #TODO-fix - inconsistent-return-statements, #TODO-fix [LOGGING] @@ -57,6 +67,9 @@ max-line-length=120 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ +# maximum number of lines in a file +max-module-lines=1100 + # Number of spaces of indent required inside a hanging or continued line. indent-after-paren=4 @@ -81,4 +94,4 @@ argument-naming-style=snake_case variable-naming-style=snake_case # Naming Style for Module -module-naming-style=snake_case \ No newline at end of file +module-naming-style=snake_case diff --git a/clients/client-python/requirements-dev.txt b/clients/client-python/requirements-dev.txt index 0d667441daa..77387c01c6c 100644 --- a/clients/client-python/requirements-dev.txt +++ b/clients/client-python/requirements-dev.txt @@ -1,8 +1,29 @@ -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + requests==2.32.2 dataclasses-json==0.6.6 pylint==3.2.2 black==24.4.2 -twine==5.1.0 +twine==5.1.1 coverage==7.5.1 +pandas==2.0.3 +pyarrow==15.0.2 +llama-index==0.10.40 +tenacity==8.3.0 +cachetools==5.3.3 +readerwriterlock==1.0.9 \ No newline at end of file diff --git a/clients/client-python/requirements.txt b/clients/client-python/requirements.txt index e4e72675b81..7242082b77c 100644 --- a/clients/client-python/requirements.txt +++ b/clients/client-python/requirements.txt @@ -1,6 +1,25 @@ -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # the tools to publish the python client to Pypi -requests -dataclasses-json \ No newline at end of file +requests==2.32.2 +dataclasses-json==0.6.6 +readerwriterlock==1.0.9 +fsspec==2024.3.1 +pyarrow==15.0.2 +cachetools==5.3.3 \ No newline at end of file diff --git a/clients/client-python/scripts/__init__.py b/clients/client-python/scripts/__init__.py new file mode 100644 index 00000000000..c206137f175 --- /dev/null +++ b/clients/client-python/scripts/__init__.py @@ -0,0 +1,18 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" diff --git a/clients/client-python/scripts/generate_doc.py b/clients/client-python/scripts/generate_doc.py new file mode 100644 index 00000000000..5bad9540906 --- /dev/null +++ b/clients/client-python/scripts/generate_doc.py @@ -0,0 +1,42 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import pydoc +import os +import shutil + +from gravitino.constants.doc import DOC_DIR +from gravitino.constants.root import GRAVITINO_DIR, MODULE_NAME + +if __name__ == "__main__": + + if os.path.exists(DOC_DIR): + # If doc folder exists, delete it + shutil.rmtree(DOC_DIR) + + # Create a new doc folder + os.makedirs(DOC_DIR) + # Change work directory to doc folder + os.chdir(DOC_DIR) + + # Write doc for top module + pydoc.writedoc(MODULE_NAME) + + # Write doc for submodules + pydoc.writedocs(GRAVITINO_DIR.as_posix(), MODULE_NAME + ".") diff --git a/clients/client-python/scripts/generate_version.py b/clients/client-python/scripts/generate_version.py new file mode 100644 index 00000000000..262bac5d1cb --- /dev/null +++ b/clients/client-python/scripts/generate_version.py @@ -0,0 +1,79 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import re +import configparser +import subprocess +from datetime import datetime + +from gravitino.constants.version import Version, VERSION_INI, SETUP_FILE +from gravitino.exceptions.gravitino_runtime_exception import GravitinoRuntimeException + +VERSION_PATTERN = r"version\s*=\s*['\"]([^'\"]+)['\"]" + + +def main(): + with open(SETUP_FILE, "r", encoding="utf-8") as f: + setup_content = f.read() + m = re.search(VERSION_PATTERN, setup_content) + if m is not None: + version = m.group(1) + else: + raise GravitinoRuntimeException("Can't find valid version info in setup.py") + + git_commit = ( + subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip() + ) + + compile_date = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + + config = configparser.ConfigParser() + config.optionxform = str + config["metadata"] = { + Version.VERSION.value: version, + Version.GIT_COMMIT.value: git_commit, + Version.COMPILE_DATE.value: compile_date, + } + + license_header = [ + "# Licensed to the Apache Software Foundation (ASF) under one\n", + "# or more contributor license agreements. See the NOTICE file\n", + "# distributed with this work for additional information\n", + "# regarding copyright ownership. The ASF licenses this file\n", + "# to you under the Apache License, Version 2.0 (the\n", + '# "License"); you may not use this file except in compliance\n', + "# with the License. You may obtain a copy of the License at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing,\n", + "# software distributed under the License is distributed on an\n", + '# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n', + "# KIND, either express or implied. See the License for the\n", + "# specific language governing permissions and limitations\n", + "# under the License.\n", + ] + + with open(VERSION_INI, "w", encoding="utf-8") as f: + f.writelines(license_header) + config.write(f) + + +if __name__ == "__main__": + main() diff --git a/clients/client-python/setup.py b/clients/client-python/setup.py index 9ec5d4a13ea..d812e593cc1 100644 --- a/clients/client-python/setup.py +++ b/clients/client-python/setup.py @@ -1,8 +1,23 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ + from setuptools import find_packages, setup @@ -22,7 +37,7 @@ author="datastrato", author_email="support@datastrato.com", python_requires=">=3.8", - packages=find_packages(exclude=["tests*"]), + packages=find_packages(exclude=["tests*", "scripts*"]), classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', diff --git a/clients/client-python/tests/__init__.py b/clients/client-python/tests/__init__.py index 5779a3ad252..c206137f175 100644 --- a/clients/client-python/tests/__init__.py +++ b/clients/client-python/tests/__init__.py @@ -1,4 +1,18 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ diff --git a/clients/client-python/tests/integration/__init__.py b/clients/client-python/tests/integration/__init__.py index 5779a3ad252..c206137f175 100644 --- a/clients/client-python/tests/integration/__init__.py +++ b/clients/client-python/tests/integration/__init__.py @@ -1,4 +1,18 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ diff --git a/clients/client-python/tests/integration/integration_test_env.py b/clients/client-python/tests/integration/integration_test_env.py index c9d0e722667..a34bba23dbb 100644 --- a/clients/client-python/tests/integration/integration_test_env.py +++ b/clients/client-python/tests/integration/integration_test_env.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ import logging diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py new file mode 100644 index 00000000000..3f8d2ae4b53 --- /dev/null +++ b/clients/client-python/tests/integration/test_catalog.py @@ -0,0 +1,90 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import logging +from random import randint + +from gravitino import ( + NameIdentifier, + GravitinoAdminClient, + GravitinoClient, + Catalog, +) + +from tests.integration.integration_test_env import IntegrationTestEnv + +logger = logging.getLogger(__name__) + + +class TestCatalog(IntegrationTestEnv): + metalake_name: str = "TestSchema_metalake" + str(randint(1, 10000)) + + catalog_name: str = "testCatalog" + catalog_location_prop: str = "location" # Fileset Catalog must set `location` + catalog_provider: str = "hadoop" + + catalog_ident: NameIdentifier = NameIdentifier.of(metalake_name, catalog_name) + + gravitino_admin_client: GravitinoAdminClient = GravitinoAdminClient( + uri="http://localhost:8090" + ) + gravitino_client: GravitinoClient = None + + def setUp(self): + self.init_test_env() + + def tearDown(self): + self.clean_test_data() + + def init_test_env(self): + self.gravitino_admin_client.create_metalake( + self.metalake_name, comment="", properties={} + ) + self.gravitino_client = GravitinoClient( + uri="http://localhost:8090", metalake_name=self.metalake_name + ) + self.gravitino_client.create_catalog( + name=self.catalog_name, + catalog_type=Catalog.Type.FILESET, + provider=self.catalog_provider, + comment="", + properties={self.catalog_location_prop: "/tmp/test_schema"}, + ) + + def clean_test_data(self): + try: + self.gravitino_client = GravitinoClient( + uri="http://localhost:8090", metalake_name=self.metalake_name + ) + logger.info( + "Drop catalog %s[%s]", + self.catalog_ident, + self.gravitino_client.drop_catalog(name=self.catalog_name), + ) + logger.info( + "Drop metalake %s[%s]", + self.metalake_name, + self.gravitino_admin_client.drop_metalake(self.metalake_name), + ) + except Exception as e: + logger.error("Clean test data failed: %s", e) + + def test_list_catalogs(self): + catalog_names = self.gravitino_client.list_catalogs() + self.assertTrue(self.catalog_name in catalog_names) diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index e299bada2b6..da79ef1ed0e 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ import logging @@ -43,19 +57,12 @@ class TestFilesetCatalog(IntegrationTestEnv): } fileset_new_name = fileset_name + "_new" - metalake_ident: NameIdentifier = NameIdentifier.of(metalake_name) - catalog_ident: NameIdentifier = NameIdentifier.of_catalog( - metalake_name, catalog_name - ) - schema_ident: NameIdentifier = NameIdentifier.of_schema( + catalog_ident: NameIdentifier = NameIdentifier.of(metalake_name, catalog_name) + schema_ident: NameIdentifier = NameIdentifier.of( metalake_name, catalog_name, schema_name ) - fileset_ident: NameIdentifier = NameIdentifier.of_fileset( - metalake_name, catalog_name, schema_name, fileset_name - ) - fileset_new_ident: NameIdentifier = NameIdentifier.of_fileset( - metalake_name, catalog_name, schema_name, fileset_new_name - ) + fileset_ident: NameIdentifier = NameIdentifier.of(schema_name, fileset_name) + fileset_new_ident: NameIdentifier = NameIdentifier.of(schema_name, fileset_new_name) gravitino_admin_client: GravitinoAdminClient = GravitinoAdminClient( uri="http://localhost:8090" @@ -73,7 +80,7 @@ def clean_test_data(self): self.gravitino_client = GravitinoClient( uri="http://localhost:8090", metalake_name=self.metalake_name ) - catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + catalog = self.gravitino_client.load_catalog(name=self.catalog_name) logger.info( "Drop fileset %s[%s]", self.fileset_ident, @@ -87,41 +94,43 @@ def clean_test_data(self): logger.info( "Drop schema %s[%s]", self.schema_ident, - catalog.as_schemas().drop_schema(ident=self.schema_ident, cascade=True), + catalog.as_schemas().drop_schema( + schema_name=self.schema_name, cascade=True + ), ) logger.info( "Drop catalog %s[%s]", self.catalog_ident, - self.gravitino_client.drop_catalog(ident=self.catalog_ident), + self.gravitino_client.drop_catalog(name=self.catalog_name), ) logger.info( "Drop metalake %s[%s]", - self.metalake_ident, - self.gravitino_admin_client.drop_metalake(self.metalake_ident), + self.metalake_name, + self.gravitino_admin_client.drop_metalake(self.metalake_name), ) except Exception as e: logger.error("Clean test data failed: %s", e) def init_test_env(self): self.gravitino_admin_client.create_metalake( - ident=self.metalake_ident, comment="", properties={} + self.metalake_name, comment="", properties={} ) self.gravitino_client = GravitinoClient( uri="http://localhost:8090", metalake_name=self.metalake_name ) catalog = self.gravitino_client.create_catalog( - ident=self.catalog_ident, + name=self.catalog_name, catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, comment="", properties={self.catalog_location_prop: "/tmp/test1"}, ) catalog.as_schemas().create_schema( - ident=self.schema_ident, comment="", properties={} + schema_name=self.schema_name, comment="", properties={} ) def create_fileset(self) -> Fileset: - catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + catalog = self.gravitino_client.load_catalog(name=self.catalog_name) return catalog.as_fileset_catalog().create_fileset( ident=self.fileset_ident, fileset_type=Fileset.Type.MANAGED, @@ -139,14 +148,14 @@ def test_create_fileset(self): def test_drop_fileset(self): self.create_fileset() - catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + catalog = self.gravitino_client.load_catalog(name=self.catalog_name) self.assertTrue( catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_ident) ) def test_list_fileset(self): self.create_fileset() - catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + catalog = self.gravitino_client.load_catalog(name=self.catalog_name) fileset_list: List[NameIdentifier] = catalog.as_fileset_catalog().list_filesets( namespace=self.fileset_ident.namespace() ) @@ -155,7 +164,7 @@ def test_list_fileset(self): def test_load_fileset(self): self.create_fileset() fileset = ( - self.gravitino_client.load_catalog(ident=self.catalog_ident) + self.gravitino_client.load_catalog(name=self.catalog_name) .as_fileset_catalog() .load_fileset(ident=self.fileset_ident) ) @@ -175,7 +184,7 @@ def test_alter_fileset(self): self.fileset_properties_key2, fileset_propertie_new_value ), ) - catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + catalog = self.gravitino_client.load_catalog(name=self.catalog_name) fileset_new = catalog.as_fileset_catalog().alter_fileset( self.fileset_ident, *changes ) diff --git a/clients/client-python/tests/integration/test_metalake.py b/clients/client-python/tests/integration/test_metalake.py index c8bbb17693f..fd8fe29efe7 100644 --- a/clients/client-python/tests/integration/test_metalake.py +++ b/clients/client-python/tests/integration/test_metalake.py @@ -1,17 +1,26 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ import logging from typing import Dict, List -from gravitino import ( - GravitinoAdminClient, - GravitinoMetalake, - MetalakeChange, - NameIdentifier, -) +from gravitino import GravitinoAdminClient, GravitinoMetalake, MetalakeChange from gravitino.dto.dto_converters import DTOConverters from gravitino.dto.requests.metalake_updates_request import MetalakeUpdatesRequest from tests.integration.integration_test_env import IntegrationTestEnv @@ -61,7 +70,7 @@ def test_create_metalake(self): def create_metalake(self, metalake_name) -> GravitinoMetalake: return self.gravitino_admin_client.create_metalake( - NameIdentifier.of(metalake_name), + metalake_name, self.metalake_comment, self.metalake_properties, ) @@ -83,7 +92,7 @@ def test_alter_metalake(self): ) metalake = self.gravitino_admin_client.alter_metalake( - NameIdentifier.of(self.metalake_name), *changes + self.metalake_name, *changes ) self.assertEqual(metalake.name(), metalake_new_name) self.assertEqual(metalake.comment(), metalake_new_comment) @@ -94,8 +103,7 @@ def test_alter_metalake(self): self.assertTrue(self.metalake_properties_key1 not in metalake.properties()) def drop_metalake(self, metalake_name: str) -> bool: - ident = NameIdentifier.of(metalake_name) - return self.gravitino_admin_client.drop_metalake(ident) + return self.gravitino_admin_client.drop_metalake(metalake_name) def test_drop_metalake(self): self.create_metalake(self.metalake_name) @@ -125,9 +133,7 @@ def test_list_metalakes(self): def test_load_metalakes(self): self.create_metalake(self.metalake_name) - metalake = self.gravitino_admin_client.load_metalake( - NameIdentifier.of(self.metalake_name) - ) + metalake = self.gravitino_admin_client.load_metalake(self.metalake_name) self.assertIsNotNone(metalake) self.assertEqual(metalake.name(), self.metalake_name) self.assertEqual(metalake.comment(), self.metalake_comment) diff --git a/clients/client-python/tests/integration/test_schema.py b/clients/client-python/tests/integration/test_schema.py index 4ad7f3fa32c..80fbff6feac 100644 --- a/clients/client-python/tests/integration/test_schema.py +++ b/clients/client-python/tests/integration/test_schema.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ import logging @@ -41,14 +55,11 @@ class TestSchema(IntegrationTestEnv): schema_properties_key2: schema_properties_value2, } - metalake_ident: NameIdentifier = NameIdentifier.of(metalake_name) - catalog_ident: NameIdentifier = NameIdentifier.of_catalog( - metalake_name, catalog_name - ) - schema_ident: NameIdentifier = NameIdentifier.of_schema( + catalog_ident: NameIdentifier = NameIdentifier.of(metalake_name, catalog_name) + schema_ident: NameIdentifier = NameIdentifier.of( metalake_name, catalog_name, schema_name ) - schema_new_ident: NameIdentifier = NameIdentifier.of_schema( + schema_new_ident: NameIdentifier = NameIdentifier.of( metalake_name, catalog_name, schema_new_name ) @@ -65,13 +76,13 @@ def tearDown(self): def init_test_env(self): self.gravitino_admin_client.create_metalake( - ident=self.metalake_ident, comment="", properties={} + self.metalake_name, comment="", properties={} ) self.gravitino_client = GravitinoClient( uri="http://localhost:8090", metalake_name=self.metalake_name ) self.gravitino_client.create_catalog( - ident=self.catalog_ident, + name=self.catalog_name, catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, comment="", @@ -83,34 +94,34 @@ def clean_test_data(self): self.gravitino_client = GravitinoClient( uri="http://localhost:8090", metalake_name=self.metalake_name ) - catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + catalog = self.gravitino_client.load_catalog(name=self.catalog_name) logger.info( "Drop schema %s[%s]", self.schema_ident, - catalog.as_schemas().drop_schema(self.schema_ident, cascade=True), + catalog.as_schemas().drop_schema(self.schema_name, cascade=True), ) logger.info( "Drop schema %s[%s]", self.schema_new_ident, - catalog.as_schemas().drop_schema(self.schema_new_ident, cascade=True), + catalog.as_schemas().drop_schema(self.schema_new_name, cascade=True), ) logger.info( "Drop catalog %s[%s]", self.catalog_ident, - self.gravitino_client.drop_catalog(ident=self.catalog_ident), + self.gravitino_client.drop_catalog(name=self.catalog_name), ) logger.info( "Drop metalake %s[%s]", - self.metalake_ident, - self.gravitino_admin_client.drop_metalake(self.metalake_ident), + self.metalake_name, + self.gravitino_admin_client.drop_metalake(self.metalake_name), ) except Exception as e: logger.error("Clean test data failed: %s", e) def create_schema(self) -> Schema: - catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + catalog = self.gravitino_client.load_catalog(name=self.catalog_name) return catalog.as_schemas().create_schema( - ident=self.schema_ident, + schema_name=self.schema_name, comment=self.schema_comment, properties=self.schema_properties, ) @@ -124,23 +135,23 @@ def test_create_schema(self): def test_drop_schema(self): self.create_schema() - catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + catalog = self.gravitino_client.load_catalog(name=self.catalog_name) self.assertTrue( - catalog.as_schemas().drop_schema(ident=self.schema_ident, cascade=True) + catalog.as_schemas().drop_schema(schema_name=self.schema_name, cascade=True) ) def test_list_schema(self): self.create_schema() - catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) - schema_list: List[NameIdentifier] = catalog.as_schemas().list_schemas( - namespace=self.schema_ident.namespace() + catalog = self.gravitino_client.load_catalog(name=self.catalog_name) + schema_list: List[str] = catalog.as_schemas().list_schemas() + self.assertTrue( + any(schema_name == self.schema_name for schema_name in schema_list) ) - self.assertTrue(any(item.name() == self.schema_name for item in schema_list)) def test_load_schema(self): self.create_schema() - catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) - schema = catalog.as_schemas().load_schema(ident=self.schema_ident) + catalog = self.gravitino_client.load_catalog(name=self.catalog_name) + schema = catalog.as_schemas().load_schema(schema_name=self.schema_name) self.assertIsNotNone(schema) self.assertEqual(schema.name(), self.schema_name) self.assertEqual(schema.comment(), self.schema_comment) @@ -157,8 +168,8 @@ def test_alter_schema(self): self.schema_properties_key2, schema_propertie_new_value ), ) - catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) - schema_new = catalog.as_schemas().alter_schema(self.schema_ident, *changes) + catalog = self.gravitino_client.load_catalog(name=self.catalog_name) + schema_new = catalog.as_schemas().alter_schema(self.schema_name, *changes) self.assertEqual( schema_new.properties().get(self.schema_properties_key2), schema_propertie_new_value, diff --git a/clients/client-python/tests/integration/test_simple_auth_client.py b/clients/client-python/tests/integration/test_simple_auth_client.py new file mode 100644 index 00000000000..516db2add06 --- /dev/null +++ b/clients/client-python/tests/integration/test_simple_auth_client.py @@ -0,0 +1,147 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import logging +import os +import unittest +from random import randint +from typing import Dict + +from gravitino import ( + NameIdentifier, + GravitinoClient, + GravitinoAdminClient, + Catalog, + Fileset, +) +from gravitino.auth.simple_auth_provider import SimpleAuthProvider + +logger = logging.getLogger(__name__) + + +class TestSimpleAuthClient(unittest.TestCase): + creator: str = "test_client" + metalake_name: str = "TestClient_metalake" + str(randint(1, 10000)) + catalog_name: str = "fileset_catalog" + catalog_location_prop: str = "location" # Fileset Catalog must set `location` + catalog_provider: str = "hadoop" + + schema_name: str = "fileset_schema" + + fileset_name: str = "test_client_fileset" + fileset_comment: str = "fileset_comment" + + fileset_location: str = "/tmp/TestFilesetCatalog" + fileset_properties_key1: str = "fileset_properties_key1" + fileset_properties_value1: str = "fileset_properties_value1" + fileset_properties_key2: str = "fileset_properties_key2" + fileset_properties_value2: str = "fileset_properties_value2" + fileset_properties: Dict[str, str] = { + fileset_properties_key1: fileset_properties_value1, + fileset_properties_key2: fileset_properties_value2, + } + + catalog_ident: NameIdentifier = NameIdentifier.of(metalake_name, catalog_name) + schema_ident: NameIdentifier = NameIdentifier.of( + metalake_name, catalog_name, schema_name + ) + fileset_ident: NameIdentifier = NameIdentifier.of(schema_name, fileset_name) + + def setUp(self): + os.environ["GRAVITINO_USER"] = self.creator + self.gravitino_admin_client = GravitinoAdminClient( + uri="http://localhost:8090", auth_data_provider=SimpleAuthProvider() + ) + self.init_test_env() + + def tearDown(self): + self.clean_test_data() + + def clean_test_data(self): + try: + catalog = self.gravitino_client.load_catalog(name=self.catalog_name) + logger.info( + "Drop fileset %s[%s]", + self.fileset_ident, + catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_ident), + ) + logger.info( + "Drop schema %s[%s]", + self.schema_ident, + catalog.as_schemas().drop_schema( + schema_name=self.schema_name, cascade=True + ), + ) + logger.info( + "Drop catalog %s[%s]", + self.catalog_ident, + self.gravitino_client.drop_catalog(name=self.catalog_name), + ) + logger.info( + "Drop metalake %s[%s]", + self.metalake_name, + self.gravitino_admin_client.drop_metalake(self.metalake_name), + ) + except Exception as e: + logger.error("Clean test data failed: %s", e) + + def init_test_env(self): + self.gravitino_admin_client.create_metalake( + self.metalake_name, comment="", properties={} + ) + self.gravitino_client = GravitinoClient( + uri="http://localhost:8090", + metalake_name=self.metalake_name, + auth_data_provider=SimpleAuthProvider(), + ) + catalog = self.gravitino_client.create_catalog( + name=self.catalog_name, + catalog_type=Catalog.Type.FILESET, + provider=self.catalog_provider, + comment="", + properties={self.catalog_location_prop: "/tmp/test1"}, + ) + catalog.as_schemas().create_schema( + schema_name=self.schema_name, comment="", properties={} + ) + catalog.as_fileset_catalog().create_fileset( + ident=self.fileset_ident, + fileset_type=Fileset.Type.MANAGED, + comment=self.fileset_comment, + storage_location=self.fileset_location, + properties=self.fileset_properties, + ) + + def test_metalake_creator(self): + metalake = self.gravitino_admin_client.load_metalake(self.metalake_name) + self.assertEqual(metalake.audit_info().creator(), self.creator) + + def test_catalog_creator(self): + catalog = self.gravitino_client.load_catalog(self.catalog_name) + self.assertEqual(catalog.audit_info().creator(), self.creator) + + def test_schema_creator(self): + catalog = self.gravitino_client.load_catalog(self.catalog_name) + schema = catalog.as_schemas().load_schema(self.schema_name) + self.assertEqual(schema.audit_info().creator(), self.creator) + + def test_fileset_creator(self): + catalog = self.gravitino_client.load_catalog(self.catalog_name) + fileset = catalog.as_fileset_catalog().load_fileset(self.fileset_ident) + self.assertEqual(fileset.audit_info().creator(), self.creator) diff --git a/clients/client-python/tests/unittests/__init__.py b/clients/client-python/tests/unittests/__init__.py new file mode 100644 index 00000000000..c206137f175 --- /dev/null +++ b/clients/client-python/tests/unittests/__init__.py @@ -0,0 +1,18 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" diff --git a/clients/client-python/tests/unittests/mock_base.py b/clients/client-python/tests/unittests/mock_base.py new file mode 100644 index 00000000000..6ad8f281e0e --- /dev/null +++ b/clients/client-python/tests/unittests/mock_base.py @@ -0,0 +1,110 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import json +from unittest.mock import patch + +from gravitino import GravitinoMetalake, Catalog, Fileset +from gravitino.catalog.fileset_catalog import FilesetCatalog +from gravitino.dto.fileset_dto import FilesetDTO +from gravitino.dto.audit_dto import AuditDTO +from gravitino.dto.metalake_dto import MetalakeDTO +from gravitino.namespace import Namespace +from gravitino.utils.http_client import HTTPClient + + +def mock_load_metalake(): + audit_dto = AuditDTO( + _creator="test", + _create_time="2022-01-01T00:00:00Z", + _last_modifier="test", + _last_modified_time="2024-04-05T10:10:35.218Z", + ) + metalake_dto = MetalakeDTO( + _name="metalake_demo", + _comment="this is test", + _properties={"k": "v"}, + _audit=audit_dto, + ) + return GravitinoMetalake(metalake_dto) + + +def mock_load_fileset_catalog(): + audit_dto = AuditDTO( + _creator="test", + _create_time="2022-01-01T00:00:00Z", + _last_modifier="test", + _last_modified_time="2024-04-05T10:10:35.218Z", + ) + + namespace = Namespace.of("metalake_demo") + + catalog = FilesetCatalog( + namespace=namespace, + name="fileset_catalog", + catalog_type=Catalog.Type.FILESET, + provider="hadoop", + comment="this is test", + properties={"k": "v"}, + audit=audit_dto, + rest_client=HTTPClient("http://localhost:9090", is_debug=True), + ) + return catalog + + +def mock_load_fileset(name: str, location: str): + audit_dto = AuditDTO( + _creator="test", + _create_time="2022-01-01T00:00:00Z", + _last_modifier="test", + _last_modified_time="2024-04-05T10:10:35.218Z", + ) + fileset = FilesetDTO( + _name=name, + _type=Fileset.Type.MANAGED, + _comment="this is test", + _properties={"k": "v"}, + _storage_location=location, + _audit=audit_dto, + ) + return fileset + + +def mock_data(cls): + @patch( + "gravitino.client.gravitino_client_base.GravitinoClientBase.load_metalake", + return_value=mock_load_metalake(), + ) + @patch( + "gravitino.client.gravitino_metalake.GravitinoMetalake.load_catalog", + return_value=mock_load_fileset_catalog(), + ) + @patch( + "gravitino.client.gravitino_client_base.GravitinoClientBase.check_version", + return_value=True, + ) + class Wrapper(cls): + pass + + return Wrapper + + +def mock_name_identifier_json(name, namespace): + + return json.dumps({"name": name, "namespace": namespace}).encode("utf-8") diff --git a/clients/client-python/tests/unittests/test_gravitino_version.py b/clients/client-python/tests/unittests/test_gravitino_version.py new file mode 100644 index 00000000000..02196effe17 --- /dev/null +++ b/clients/client-python/tests/unittests/test_gravitino_version.py @@ -0,0 +1,104 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import unittest + +from gravitino.client.gravitino_version import GravitinoVersion +from gravitino.dto.version_dto import VersionDTO +from gravitino.exceptions.gravitino_runtime_exception import GravitinoRuntimeException + + +class TestGravitinoVersion(unittest.TestCase): + def test_parse_version_string(self): + # Test a valid the version string + version = GravitinoVersion(VersionDTO("0.6.0", "2023-01-01", "1234567")) + + self.assertEqual(version.major, 0) + self.assertEqual(version.minor, 6) + self.assertEqual(version.patch, 0) + + # Test a valid the version string with SNAPSHOT + version = GravitinoVersion( + VersionDTO("0.6.0-SNAPSHOT", "2023-01-01", "1234567") + ) + + self.assertEqual(version.major, 0) + self.assertEqual(version.minor, 6) + self.assertEqual(version.patch, 0) + + # Test a valid the version string with alpha + version = GravitinoVersion(VersionDTO("0.6.0-alpha", "2023-01-01", "1234567")) + + self.assertEqual(version.major, 0) + self.assertEqual(version.minor, 6) + self.assertEqual(version.patch, 0) + + # Test a valid the version string with pypi format + version = GravitinoVersion(VersionDTO("0.6.0.dev21", "2023-01-01", "1234567")) + + self.assertEqual(version.major, 0) + self.assertEqual(version.minor, 6) + self.assertEqual(version.patch, 0) + + # Test an invalid the version string with 2 part + self.assertRaises( + AssertionError, GravitinoVersion, VersionDTO("0.6", "2023-01-01", "1234567") + ) + + # Test an invalid the version string with not number + self.assertRaises( + AssertionError, + GravitinoVersion, + VersionDTO("a.b.c", "2023-01-01", "1234567"), + ) + + def test_version_compare(self): + # test equal + version1 = GravitinoVersion(VersionDTO("0.6.0", "2023-01-01", "1234567")) + version2 = GravitinoVersion(VersionDTO("0.6.0", "2023-01-01", "1234567")) + + self.assertEqual(version1, version2) + + # test less than + version1 = GravitinoVersion(VersionDTO("0.6.0", "2023-01-01", "1234567")) + version2 = GravitinoVersion(VersionDTO("0.12.0", "2023-01-01", "1234567")) + + self.assertLess(version1, version2) + + # test greater than + version1 = GravitinoVersion(VersionDTO("1.6.0", "2023-01-01", "1234567")) + version2 = GravitinoVersion(VersionDTO("0.6.0", "2023-01-01", "1234567")) + + self.assertGreater(version1, version2) + + # test equal with suffix + version1 = GravitinoVersion( + VersionDTO("0.6.0-SNAPSHOT", "2023-01-01", "1234567") + ) + version2 = GravitinoVersion(VersionDTO("0.6.0", "2023-01-01", "1234567")) + + self.assertEqual(version1, version2) + + # test compare with other class + + version1 = GravitinoVersion(VersionDTO("0.6.0", "2023-01-01", "1234567")) + version2 = "0.6.0" + + self.assertRaises(GravitinoRuntimeException, version1.__eq__, version2) + self.assertRaises(GravitinoRuntimeException, version1.__gt__, version2) diff --git a/clients/client-python/tests/unittests/test_gvfs_with_local.py b/clients/client-python/tests/unittests/test_gvfs_with_local.py new file mode 100644 index 00000000000..c5cebc7644f --- /dev/null +++ b/clients/client-python/tests/unittests/test_gvfs_with_local.py @@ -0,0 +1,1005 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +# pylint: disable=protected-access + +import random +import string +import time +import unittest +from unittest.mock import patch + +import pandas +import pyarrow as pa +import pyarrow.dataset as dt +import pyarrow.parquet as pq +from fsspec.implementations.local import LocalFileSystem +from llama_index.core import SimpleDirectoryReader + +from gravitino import gvfs +from gravitino import NameIdentifier +from gravitino.dto.audit_dto import AuditDTO +from gravitino.dto.fileset_dto import FilesetDTO +from gravitino.filesystem.gvfs import FilesetContext, StorageType +from gravitino.exceptions.gravitino_runtime_exception import GravitinoRuntimeException + +from tests.unittests import mock_base + + +def generate_unique_random_string(length): + characters = string.ascii_letters + string.digits + random_string = "".join(random.sample(characters, length)) + return random_string + + +@mock_base.mock_data +class TestLocalFilesystem(unittest.TestCase): + _local_base_dir_path: str = "file:/tmp/fileset" + _fileset_dir: str = ( + f"{_local_base_dir_path}/{generate_unique_random_string(10)}/fileset_catalog/tmp" + ) + + def setUp(self) -> None: + local_fs = LocalFileSystem() + if not local_fs.exists(self._fileset_dir): + local_fs.mkdir(self._fileset_dir) + + def tearDown(self) -> None: + local_fs = LocalFileSystem() + if local_fs.exists(self._local_base_dir_path): + local_fs.rm(self._local_base_dir_path, recursive=True) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_cache", f"{_fileset_dir}/test_cache" + ), + ) + def test_cache(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_cache" + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_cache" + local_fs.mkdir(fileset_storage_location) + self.assertTrue(local_fs.exists(fileset_storage_location)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", + metalake_name="metalake_demo", + cache_size=1, + cache_expired_time=1, + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + # wait 2 seconds + time.sleep(2) + self.assertIsNone( + fs.cache.get( + NameIdentifier.of( + "metalake_demo", "fileset_catalog", "tmp", "test_cache" + ) + ) + ) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset("test_ls", f"{_fileset_dir}/test_ls"), + ) + def test_ls(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_ls" + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_ls" + local_fs.mkdir(fileset_storage_location) + sub_dir_path = f"{fileset_storage_location}/test_1" + local_fs.mkdir(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + sub_file_path = f"{fileset_storage_location}/test_file_1.par" + local_fs.touch(sub_file_path) + self.assertTrue(local_fs.exists(sub_file_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + # test detail = false + file_list_without_detail = fs.ls(fileset_virtual_location, detail=False) + file_list_without_detail.sort() + self.assertEqual(2, len(file_list_without_detail)) + self.assertEqual( + file_list_without_detail[0], f"{fileset_virtual_location}/test_1" + ) + self.assertEqual( + file_list_without_detail[1], f"{fileset_virtual_location}/test_file_1.par" + ) + + # test detail = true + file_list_with_detail = fs.ls(fileset_virtual_location, detail=True) + file_list_with_detail.sort(key=lambda x: x["name"]) + self.assertEqual(2, len(file_list_with_detail)) + self.assertEqual( + file_list_with_detail[0]["name"], f"{fileset_virtual_location}/test_1" + ) + self.assertEqual( + file_list_with_detail[1]["name"], + f"{fileset_virtual_location}/test_file_1.par", + ) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_info", f"{_fileset_dir}/test_info" + ), + ) + def test_info(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_info" + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_info" + local_fs.mkdir(fileset_storage_location) + sub_dir_path = f"{fileset_storage_location}/test_1" + local_fs.mkdir(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + sub_file_path = f"{fileset_storage_location}/test_file_1.par" + local_fs.touch(sub_file_path) + self.assertTrue(local_fs.exists(sub_file_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + dir_virtual_path = fileset_virtual_location + "/test_1" + dir_info = fs.info(dir_virtual_path) + self.assertEqual(dir_info["name"], dir_virtual_path) + + file_virtual_path = fileset_virtual_location + "/test_file_1.par" + file_info = fs.info(file_virtual_path) + self.assertEqual(file_info["name"], file_virtual_path) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_exist", f"{_fileset_dir}/test_exist" + ), + ) + def test_exist(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_exist" + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_exist" + local_fs.mkdir(fileset_storage_location) + sub_dir_path = f"{fileset_storage_location}/test_1" + local_fs.mkdir(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + sub_file_path = f"{fileset_storage_location}/test_file_1.par" + local_fs.touch(sub_file_path) + self.assertTrue(local_fs.exists(sub_file_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + dir_virtual_path = fileset_virtual_location + "/test_1" + self.assertTrue(fs.exists(dir_virtual_path)) + + file_virtual_path = fileset_virtual_location + "/test_file_1.par" + self.assertTrue(fs.exists(file_virtual_path)) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_cp_file", f"{_fileset_dir}/test_cp_file" + ), + ) + def test_cp_file(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_cp_file" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_cp_file" + sub_file_path = f"{fileset_storage_location}/test_file_1.par" + local_fs.touch(sub_file_path) + self.assertTrue(local_fs.exists(sub_file_path)) + + with local_fs.open(sub_file_path, "wb") as f: + f.write(b"test_file_1") + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + file_virtual_path = fileset_virtual_location + "/test_file_1.par" + self.assertTrue(fs.exists(file_virtual_path)) + + cp_file_virtual_path = fileset_virtual_location + "/test_cp_file_1.par" + fs.cp_file(file_virtual_path, cp_file_virtual_path) + self.assertTrue(fs.exists(cp_file_virtual_path)) + with local_fs.open(sub_file_path, "rb") as f: + result = f.read() + self.assertEqual(b"test_file_1", result) + + # test invalid dst path + cp_file_invalid_virtual_path = ( + "fileset/fileset_catalog/tmp/invalid_fileset/test_cp_file_1.par" + ) + with self.assertRaises(GravitinoRuntimeException): + fs.cp_file(file_virtual_path, cp_file_invalid_virtual_path) + + # test mount a single file + local_fs.rm(path=fileset_storage_location, recursive=True) + self.assertFalse(local_fs.exists(fileset_storage_location)) + local_fs.touch(fileset_storage_location) + self.assertTrue(local_fs.exists(fileset_storage_location)) + with self.assertRaises(GravitinoRuntimeException): + fs.cp_file(file_virtual_path, cp_file_virtual_path) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset("test_mv", f"{_fileset_dir}/test_mv"), + ) + def test_mv(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_mv" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_mv" + + sub_file_path = f"{fileset_storage_location}/test_file_1.par" + local_fs.touch(sub_file_path) + self.assertTrue(local_fs.exists(sub_file_path)) + + another_dir_path = f"{fileset_storage_location}/another_dir" + local_fs.mkdirs(another_dir_path) + self.assertTrue(local_fs.exists(another_dir_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + file_virtual_path = fileset_virtual_location + "/test_file_1.par" + self.assertTrue(fs.exists(file_virtual_path)) + + mv_file_virtual_path = fileset_virtual_location + "/test_cp_file_1.par" + fs.mv(file_virtual_path, mv_file_virtual_path) + self.assertTrue(fs.exists(mv_file_virtual_path)) + + mv_another_dir_virtual_path = ( + fileset_virtual_location + "/another_dir/test_file_2.par" + ) + fs.mv(mv_file_virtual_path, mv_another_dir_virtual_path) + self.assertTrue(fs.exists(mv_another_dir_virtual_path)) + + # test not exist dir + not_exist_dst_dir_path = fileset_virtual_location + "/not_exist/test_file_2.par" + with self.assertRaises(FileNotFoundError): + fs.mv(path1=mv_another_dir_virtual_path, path2=not_exist_dst_dir_path) + + # test invalid dst path + mv_file_invalid_virtual_path = ( + "fileset/fileset_catalog/tmp/invalid_fileset/test_cp_file_1.par" + ) + with self.assertRaises(GravitinoRuntimeException): + fs.mv(path1=file_virtual_path, path2=mv_file_invalid_virtual_path) + + # test mount a single file + local_fs.rm(path=fileset_storage_location, recursive=True) + self.assertFalse(local_fs.exists(fileset_storage_location)) + local_fs.touch(fileset_storage_location) + self.assertTrue(local_fs.exists(fileset_storage_location)) + with self.assertRaises(GravitinoRuntimeException): + fs.mv(file_virtual_path, mv_file_virtual_path) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset("test_rm", f"{_fileset_dir}/test_rm"), + ) + def test_rm(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_rm" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_rm" + + sub_file_path = f"{fileset_storage_location}/test_file_1.par" + local_fs.touch(sub_file_path) + self.assertTrue(local_fs.exists(sub_file_path)) + + sub_dir_path = f"{fileset_storage_location}/sub_dir" + local_fs.mkdirs(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + # test delete file + file_virtual_path = fileset_virtual_location + "/test_file_1.par" + self.assertTrue(fs.exists(file_virtual_path)) + fs.rm(file_virtual_path) + self.assertFalse(fs.exists(file_virtual_path)) + + # test delete dir with recursive = false + dir_virtual_path = fileset_virtual_location + "/sub_dir" + self.assertTrue(fs.exists(dir_virtual_path)) + with self.assertRaises(ValueError): + fs.rm(dir_virtual_path, recursive=False) + + # test delete dir with recursive = true + fs.rm(dir_virtual_path, recursive=True) + self.assertFalse(fs.exists(dir_virtual_path)) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_rm_file", f"{_fileset_dir}/test_rm_file" + ), + ) + def test_rm_file(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_rm_file" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_rm_file" + + sub_file_path = f"{fileset_storage_location}/test_file_1.par" + local_fs.touch(sub_file_path) + self.assertTrue(local_fs.exists(sub_file_path)) + + sub_dir_path = f"{fileset_storage_location}/sub_dir" + local_fs.mkdirs(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + # test delete file + file_virtual_path = fileset_virtual_location + "/test_file_1.par" + self.assertTrue(fs.exists(file_virtual_path)) + fs.rm_file(file_virtual_path) + self.assertFalse(fs.exists(file_virtual_path)) + + # test delete dir + dir_virtual_path = fileset_virtual_location + "/sub_dir" + self.assertTrue(fs.exists(dir_virtual_path)) + with self.assertRaises((IsADirectoryError, PermissionError)): + fs.rm_file(dir_virtual_path) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_rmdir", f"{_fileset_dir}/test_rmdir" + ), + ) + def test_rmdir(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_rmdir" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_rmdir" + + sub_file_path = f"{fileset_storage_location}/test_file_1.par" + local_fs.touch(sub_file_path) + self.assertTrue(local_fs.exists(sub_file_path)) + + sub_dir_path = f"{fileset_storage_location}/sub_dir" + local_fs.mkdirs(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + # test delete file + file_virtual_path = fileset_virtual_location + "/test_file_1.par" + self.assertTrue(fs.exists(file_virtual_path)) + with self.assertRaises(NotADirectoryError): + fs.rmdir(file_virtual_path) + + # test delete dir + dir_virtual_path = fileset_virtual_location + "/sub_dir" + self.assertTrue(fs.exists(dir_virtual_path)) + fs.rmdir(dir_virtual_path) + self.assertFalse(fs.exists(dir_virtual_path)) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_open", f"{_fileset_dir}/test_open" + ), + ) + def test_open(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_open" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_open" + + sub_file_path = f"{fileset_storage_location}/test_file_1.par" + local_fs.touch(sub_file_path) + self.assertTrue(local_fs.exists(sub_file_path)) + + sub_dir_path = f"{fileset_storage_location}/sub_dir" + local_fs.mkdirs(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + # test open and write file + file_virtual_path = fileset_virtual_location + "/test_file_1.par" + self.assertTrue(fs.exists(file_virtual_path)) + with fs.open(file_virtual_path, mode="wb") as f: + f.write(b"test_open_write") + self.assertTrue(fs.info(file_virtual_path)["size"] > 0) + + # test open and read file + with fs.open(file_virtual_path, mode="rb") as f: + self.assertEqual(b"test_open_write", f.read()) + + # test open dir + dir_virtual_path = fileset_virtual_location + "/sub_dir" + self.assertTrue(fs.exists(dir_virtual_path)) + with self.assertRaises(IsADirectoryError): + fs.open(dir_virtual_path) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_mkdir", f"{_fileset_dir}/test_mkdir" + ), + ) + def test_mkdir(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_mkdir" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_mkdir" + + sub_dir_path = f"{fileset_storage_location}/sub_dir" + local_fs.mkdirs(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + # test mkdir dir which exists + existed_dir_virtual_path = fileset_virtual_location + self.assertTrue(fs.exists(existed_dir_virtual_path)) + with self.assertRaises(FileExistsError): + fs.mkdir(existed_dir_virtual_path) + + # test mkdir dir with create_parents = false + parent_not_exist_virtual_path = fileset_virtual_location + "/not_exist/sub_dir" + self.assertFalse(fs.exists(parent_not_exist_virtual_path)) + with self.assertRaises(FileNotFoundError): + fs.mkdir(parent_not_exist_virtual_path, create_parents=False) + + # test mkdir dir with create_parents = true + parent_not_exist_virtual_path2 = fileset_virtual_location + "/not_exist/sub_dir" + self.assertFalse(fs.exists(parent_not_exist_virtual_path2)) + fs.mkdir(parent_not_exist_virtual_path2, create_parents=True) + self.assertTrue(fs.exists(parent_not_exist_virtual_path2)) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_makedirs", f"{_fileset_dir}/test_makedirs" + ), + ) + def test_makedirs(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_makedirs" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_makedirs" + + sub_dir_path = f"{fileset_storage_location}/sub_dir" + local_fs.mkdirs(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + # test mkdir dir which exists + existed_dir_virtual_path = fileset_virtual_location + self.assertTrue(fs.exists(existed_dir_virtual_path)) + with self.assertRaises(FileExistsError): + fs.mkdirs(existed_dir_virtual_path) + + # test mkdir dir not exist + parent_not_exist_virtual_path = fileset_virtual_location + "/not_exist/sub_dir" + self.assertFalse(fs.exists(parent_not_exist_virtual_path)) + fs.makedirs(parent_not_exist_virtual_path) + self.assertTrue(fs.exists(parent_not_exist_virtual_path)) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_created", f"{_fileset_dir}/test_created" + ), + ) + def test_created(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_created" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_created" + + sub_dir_path = f"{fileset_storage_location}/sub_dir" + local_fs.mkdirs(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + # test mkdir dir which exists + dir_virtual_path = fileset_virtual_location + "/sub_dir" + self.assertTrue(fs.exists(dir_virtual_path)) + self.assertIsNotNone(fs.created(dir_virtual_path)) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_modified", f"{_fileset_dir}/test_modified" + ), + ) + def test_modified(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_modified" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_modified" + + sub_dir_path = f"{fileset_storage_location}/sub_dir" + local_fs.mkdirs(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + # test mkdir dir which exists + dir_virtual_path = fileset_virtual_location + "/sub_dir" + self.assertTrue(fs.exists(dir_virtual_path)) + self.assertIsNotNone(fs.modified(dir_virtual_path)) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_cat_file", f"{_fileset_dir}/test_cat_file" + ), + ) + def test_cat_file(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_cat_file" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_cat_file" + + sub_file_path = f"{fileset_storage_location}/test_file_1.par" + local_fs.touch(sub_file_path) + self.assertTrue(local_fs.exists(sub_file_path)) + + sub_dir_path = f"{fileset_storage_location}/sub_dir" + local_fs.mkdirs(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + # test open and write file + file_virtual_path = fileset_virtual_location + "/test_file_1.par" + self.assertTrue(fs.exists(file_virtual_path)) + with fs.open(file_virtual_path, mode="wb") as f: + f.write(b"test_cat_file") + self.assertTrue(fs.info(file_virtual_path)["size"] > 0) + + # test cat file + content = fs.cat_file(file_virtual_path) + self.assertEqual(b"test_cat_file", content) + + # test cat dir + dir_virtual_path = fileset_virtual_location + "/sub_dir" + self.assertTrue(fs.exists(dir_virtual_path)) + with self.assertRaises(IsADirectoryError): + fs.cat_file(dir_virtual_path) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_get_file", f"{_fileset_dir}/test_get_file" + ), + ) + def test_get_file(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_get_file" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "fileset/fileset_catalog/tmp/test_get_file" + + sub_file_path = f"{fileset_storage_location}/test_file_1.par" + local_fs.touch(sub_file_path) + self.assertTrue(local_fs.exists(sub_file_path)) + + sub_dir_path = f"{fileset_storage_location}/sub_dir" + local_fs.mkdirs(sub_dir_path) + self.assertTrue(local_fs.exists(sub_dir_path)) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + self.assertTrue(fs.exists(fileset_virtual_location)) + + # test open and write file + file_virtual_path = fileset_virtual_location + "/test_file_1.par" + self.assertTrue(fs.exists(file_virtual_path)) + with fs.open(file_virtual_path, mode="wb") as f: + f.write(b"test_get_file") + self.assertTrue(fs.info(file_virtual_path)["size"] > 0) + + # test get file + local_path = self._fileset_dir + "/local_file.par" + local_fs.touch(local_path) + self.assertTrue(local_fs.exists(local_path)) + fs.get_file(file_virtual_path, local_path) + self.assertEqual(b"test_get_file", local_fs.cat_file(local_path)) + + # test get a dir + dir_virtual_path = fileset_virtual_location + "/sub_dir" + local_path = self._fileset_dir + "/local_dir" + self.assertTrue(fs.exists(dir_virtual_path)) + fs.get_file(dir_virtual_path, local_path) + self.assertTrue(local_fs.exists(local_path)) + + # test get a file to a remote file + remote_path = "gvfs://" + fileset_virtual_location + "/test_file_2.par" + with self.assertRaises(GravitinoRuntimeException): + fs.get_file(file_virtual_path, remote_path) + + def test_convert_actual_path(self, *mock_methods): + # test convert actual hdfs path + audit_dto = AuditDTO( + _creator="test", + _create_time="2022-01-01T00:00:00Z", + _last_modifier="test", + _last_modified_time="2024-04-05T10:10:35.218Z", + ) + hdfs_fileset: FilesetDTO = FilesetDTO( + _name="test_f1", + _comment="", + _type=FilesetDTO.Type.MANAGED, + _storage_location="hdfs://localhost:8090/fileset/test_f1", + _audit=audit_dto, + _properties={}, + ) + mock_hdfs_context: FilesetContext = FilesetContext( + name_identifier=NameIdentifier.of( + "test_metalake", "test_catalog", "test_schema", "test_f1" + ), + storage_type=StorageType.HDFS, + fileset=hdfs_fileset, + actual_path=hdfs_fileset.storage_location() + "/actual_path", + fs=LocalFileSystem(), + ) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + # test actual path not start with storage location + actual_path = "/not_start_with_storage/ttt" + with self.assertRaises(GravitinoRuntimeException): + fs._convert_actual_path(actual_path, mock_hdfs_context) + + # test actual path start with storage location + actual_path = "/fileset/test_f1/actual_path" + virtual_path = fs._convert_actual_path(actual_path, mock_hdfs_context) + self.assertEqual( + "fileset/test_catalog/test_schema/test_f1/actual_path", virtual_path + ) + + # test convert actual local path + audit_dto = AuditDTO( + _creator="test", + _create_time="2022-01-01T00:00:00Z", + _last_modifier="test", + _last_modified_time="2024-04-05T10:10:35.218Z", + ) + local_fileset: FilesetDTO = FilesetDTO( + _name="test_f1", + _comment="", + _type=FilesetDTO.Type.MANAGED, + _storage_location="file:/tmp/fileset/test_f1", + _audit=audit_dto, + _properties={}, + ) + mock_local_context: FilesetContext = FilesetContext( + name_identifier=NameIdentifier.of( + "test_metalake", "test_catalog", "test_schema", "test_f1" + ), + storage_type=StorageType.LOCAL, + fileset=local_fileset, + actual_path=local_fileset.storage_location() + "/actual_path", + fs=LocalFileSystem(), + ) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + # test actual path not start with storage location + actual_path = "/not_start_with_storage/ttt" + with self.assertRaises(GravitinoRuntimeException): + fs._convert_actual_path(actual_path, mock_local_context) + + # test actual path start with storage location + actual_path = "/tmp/fileset/test_f1/actual_path" + virtual_path = fs._convert_actual_path(actual_path, mock_local_context) + self.assertEqual( + "fileset/test_catalog/test_schema/test_f1/actual_path", virtual_path + ) + + def test_convert_info(self, *mock_methods3): + # test convert actual hdfs path + audit_dto = AuditDTO( + _creator="test", + _create_time="2022-01-01T00:00:00Z", + _last_modifier="test", + _last_modified_time="2024-04-05T10:10:35.218Z", + ) + hdfs_fileset: FilesetDTO = FilesetDTO( + _name="test_f1", + _comment="", + _type=FilesetDTO.Type.MANAGED, + _storage_location="hdfs://localhost:8090/fileset/test_f1", + _audit=audit_dto, + _properties={}, + ) + mock_hdfs_context: FilesetContext = FilesetContext( + name_identifier=NameIdentifier.of( + "test_metalake", "test_catalog", "test_schema", "test_f1" + ), + storage_type=StorageType.HDFS, + fileset=hdfs_fileset, + actual_path=hdfs_fileset.storage_location() + "/actual_path", + fs=LocalFileSystem(), + ) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + # test actual path not start with storage location + actual_path = "/not_start_with_storage/ttt" + with self.assertRaises(GravitinoRuntimeException): + fs._convert_actual_path(actual_path, mock_hdfs_context) + + # test actual path start with storage location + actual_path = "/fileset/test_f1/actual_path" + virtual_path = fs._convert_actual_path(actual_path, mock_hdfs_context) + self.assertEqual( + "fileset/test_catalog/test_schema/test_f1/actual_path", virtual_path + ) + + # test convert actual local path + audit_dto = AuditDTO( + _creator="test", + _create_time="2022-01-01T00:00:00Z", + _last_modifier="test", + _last_modified_time="2024-04-05T10:10:35.218Z", + ) + local_fileset: FilesetDTO = FilesetDTO( + _name="test_f1", + _comment="", + _type=FilesetDTO.Type.MANAGED, + _storage_location="file:/tmp/fileset/test_f1", + _audit=audit_dto, + _properties={}, + ) + mock_local_context: FilesetContext = FilesetContext( + name_identifier=NameIdentifier.of( + "test_metalake", "test_catalog", "test_schema", "test_f1" + ), + storage_type=StorageType.LOCAL, + fileset=local_fileset, + actual_path=local_fileset.storage_location() + "/actual_path", + fs=LocalFileSystem(), + ) + + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + # test actual path not start with storage location + actual_path = "/not_start_with_storage/ttt" + with self.assertRaises(GravitinoRuntimeException): + fs._convert_actual_path(actual_path, mock_local_context) + + # test actual path start with storage location + actual_path = "/tmp/fileset/test_f1/actual_path" + virtual_path = fs._convert_actual_path(actual_path, mock_local_context) + self.assertEqual( + "fileset/test_catalog/test_schema/test_f1/actual_path", virtual_path + ) + + def test_extract_identifier(self, *mock_methods): + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:9090", metalake_name="metalake_demo" + ) + with self.assertRaises(GravitinoRuntimeException): + fs._extract_identifier(path=None) + + invalid_path = "s3://bucket_1/test_catalog/schema/fileset/ttt" + with self.assertRaises(GravitinoRuntimeException): + fs._extract_identifier(path=invalid_path) + + valid_path = "fileset/test_catalog/schema/fileset/ttt" + identifier: NameIdentifier = fs._extract_identifier(path=valid_path) + self.assertEqual("metalake_demo", identifier.namespace().level(0)) + self.assertEqual("test_catalog", identifier.namespace().level(1)) + self.assertEqual("schema", identifier.namespace().level(2)) + self.assertEqual("fileset", identifier.name()) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_pandas", f"{_fileset_dir}/test_pandas" + ), + ) + def test_pandas(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_pandas" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "gvfs://fileset/fileset_catalog/tmp/test_pandas" + data = pandas.DataFrame({"Name": ["A", "B", "C", "D"], "ID": [20, 21, 19, 18]}) + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:8090", metalake_name="test_metalake" + ) + # to parquet + data.to_parquet(fileset_virtual_location + "/test.parquet", filesystem=fs) + self.assertTrue(local_fs.exists(fileset_storage_location + "/test.parquet")) + + # read parquet + ds1 = pandas.read_parquet( + path=fileset_virtual_location + "/test.parquet", filesystem=fs + ) + self.assertTrue(data.equals(ds1)) + storage_options = { + "server_uri": "http://localhost:8090", + "metalake_name": "test_metalake", + } + # to csv + data.to_csv( + fileset_virtual_location + "/test.csv", + index=False, + storage_options=storage_options, + ) + self.assertTrue(local_fs.exists(fileset_storage_location + "/test.csv")) + + # read csv + ds2 = pandas.read_csv( + fileset_virtual_location + "/test.csv", storage_options=storage_options + ) + self.assertTrue(data.equals(ds2)) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_pyarrow", f"{_fileset_dir}/test_pyarrow" + ), + ) + def test_pyarrow(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_pyarrow" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "gvfs://fileset/fileset_catalog/tmp/test_pyarrow" + data = pandas.DataFrame({"Name": ["A", "B", "C", "D"], "ID": [20, 21, 19, 18]}) + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:8090", metalake_name="test_metalake" + ) + + # to parquet + data.to_parquet(fileset_virtual_location + "/test.parquet", filesystem=fs) + self.assertTrue(local_fs.exists(fileset_storage_location + "/test.parquet")) + + # read as arrow dataset + arrow_dataset = dt.dataset( + fileset_virtual_location + "/test.parquet", filesystem=fs + ) + arrow_tb_1 = arrow_dataset.to_table() + + arrow_tb_2 = pa.Table.from_pandas(data) + self.assertTrue(arrow_tb_1.equals(arrow_tb_2)) + + # read as arrow parquet dataset + arrow_tb_3 = pq.read_table( + fileset_virtual_location + "/test.parquet", filesystem=fs + ) + self.assertTrue(arrow_tb_3.equals(arrow_tb_2)) + + @patch( + "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset", + return_value=mock_base.mock_load_fileset( + "test_llama_index", f"{_fileset_dir}/test_llama_index" + ), + ) + def test_llama_index(self, *mock_methods): + local_fs = LocalFileSystem() + fileset_storage_location = f"{self._fileset_dir}/test_llama_index" + local_fs.mkdir(fileset_storage_location) + + fileset_virtual_location = "gvfs://fileset/fileset_catalog/tmp/test_llama_index" + data = pandas.DataFrame({"Name": ["A", "B", "C", "D"], "ID": [20, 21, 19, 18]}) + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:8090", metalake_name="test_metalake" + ) + + storage_options = { + "server_uri": "http://localhost:8090", + "metalake_name": "test_metalake", + } + # to csv + data.to_csv( + fileset_virtual_location + "/test.csv", + index=False, + storage_options=storage_options, + ) + self.assertTrue(local_fs.exists(fileset_storage_location + "/test.csv")) + + data.to_csv( + fileset_virtual_location + "/sub_dir/test1.csv", + index=False, + storage_options=storage_options, + ) + self.assertTrue( + local_fs.exists(fileset_storage_location + "/sub_dir/test1.csv") + ) + + reader = SimpleDirectoryReader( + input_dir="fileset/fileset_catalog/tmp/test_llama_index", + fs=fs, + recursive=True, # recursively searches all subdirectories + ) + documents = reader.load_data() + self.assertEqual(len(documents), 2) + doc_1 = documents[0] + result_1 = [line.strip().split(", ") for line in doc_1.text.split("\n")] + self.assertEqual(4, len(result_1)) + for row in result_1: + if row[0] == "A": + self.assertEqual(row[1], "20") + elif row[0] == "B": + self.assertEqual(row[1], "21") + elif row[0] == "C": + self.assertEqual(row[1], "19") + elif row[0] == "D": + self.assertEqual(row[1], "18") diff --git a/clients/client-python/tests/unittests/test_metalake.py b/clients/client-python/tests/unittests/test_metalake.py index 3c465a2530c..b4ca90c0518 100644 --- a/clients/client-python/tests/unittests/test_metalake.py +++ b/clients/client-python/tests/unittests/test_metalake.py @@ -1,6 +1,20 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ import unittest @@ -17,10 +31,10 @@ def test_from_json_metalake_response(self): ) metalake_response = MetalakeResponse.from_json(str_json, infer_missing=True) self.assertEqual(metalake_response.code(), 0) - self.assertIsNotNone(metalake_response._metalake) - self.assertEqual(metalake_response._metalake.name(), "example_name18") + self.assertIsNotNone(metalake_response.metalake()) + self.assertEqual(metalake_response.metalake().name(), "example_name18") self.assertEqual( - metalake_response._metalake.audit_info().creator(), "anonymous" + metalake_response.metalake().audit_info().creator(), "anonymous" ) def test_from_error_json_metalake_response(self): @@ -32,8 +46,8 @@ def test_from_error_json_metalake_response(self): ) metalake_response = MetalakeResponse.from_json(str_json, infer_missing=True) self.assertEqual(metalake_response.code(), 0) - self.assertIsNotNone(metalake_response._metalake) - self.assertEqual(metalake_response._metalake.name(), "example_name18") + self.assertIsNotNone(metalake_response.metalake()) + self.assertEqual(metalake_response.metalake().name(), "example_name18") self.assertEqual( - metalake_response._metalake.audit_info().creator(), "anonymous" + metalake_response.metalake().audit_info().creator(), "anonymous" ) diff --git a/clients/client-python/tests/unittests/test_name_identifier.py b/clients/client-python/tests/unittests/test_name_identifier.py index 11c08bd478d..574a68ddb5b 100644 --- a/clients/client-python/tests/unittests/test_name_identifier.py +++ b/clients/client-python/tests/unittests/test_name_identifier.py @@ -1,22 +1,49 @@ """ -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. """ import unittest from gravitino import NameIdentifier +from tests.unittests.mock_base import mock_name_identifier_json class TestNameIdentifier(unittest.TestCase): def test_name_identifier_hash(self): - name_identifier1: NameIdentifier = NameIdentifier.of_fileset( + name_identifier1: NameIdentifier = NameIdentifier.of( "test_metalake", "test_catalog", "test_schema", "test_fileset1" ) - name_identifier2: NameIdentifier = NameIdentifier.of_fileset( + name_identifier2: NameIdentifier = NameIdentifier.of( "test_metalake", "test_catalog", "test_schema", "test_fileset2" ) identifier_dict = {name_identifier1: "test1", name_identifier2: "test2"} self.assertEqual("test1", identifier_dict.get(name_identifier1)) self.assertNotEqual("test2", identifier_dict.get(name_identifier1)) + + def test_from_json_name_identifier(self): + + test_name = "test_name_identifier" + test_namespace_levels = ["1", "2", "3", "4"] + + json_data = mock_name_identifier_json(test_name, test_namespace_levels) + + name_identifier = NameIdentifier.from_json(json_data, infer_missing=True) + + self.assertEqual(test_name, name_identifier.name()) + self.assertEqual(test_namespace_levels, name_identifier.namespace().levels()) diff --git a/clients/client-python/tests/unittests/test_simple_auth_provider.py b/clients/client-python/tests/unittests/test_simple_auth_provider.py new file mode 100644 index 00000000000..d8c10e4671a --- /dev/null +++ b/clients/client-python/tests/unittests/test_simple_auth_provider.py @@ -0,0 +1,52 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import base64 +import os +import unittest + +from gravitino.auth.auth_constants import AuthConstants +from gravitino.auth.auth_data_provider import AuthDataProvider +from gravitino.auth.simple_auth_provider import SimpleAuthProvider + + +class TestSimpleAuthProvider(unittest.TestCase): + + def test_auth_provider(self): + os.environ["user.name"] = "test_auth1" + provider: AuthDataProvider = SimpleAuthProvider() + self.assertTrue(provider.has_token_data()) + user: str = os.environ["user.name"] + token = provider.get_token_data().decode("utf-8") + self.assertTrue(token.startswith(AuthConstants.AUTHORIZATION_BASIC_HEADER)) + token_string = base64.b64decode( + token[len(AuthConstants.AUTHORIZATION_BASIC_HEADER) :] + ).decode("utf-8") + self.assertEqual(f"{user}:dummy", token_string) + + os.environ["GRAVITINO_USER"] = "test_auth2" + provider: AuthDataProvider = SimpleAuthProvider() + self.assertTrue(provider.has_token_data()) + user: str = os.environ["GRAVITINO_USER"] + token = provider.get_token_data().decode("utf-8") + self.assertTrue(token.startswith(AuthConstants.AUTHORIZATION_BASIC_HEADER)) + token_string = base64.b64decode( + token[len(AuthConstants.AUTHORIZATION_BASIC_HEADER) :] + ).decode("utf-8") + self.assertEqual(f"{user}:dummy", token_string) diff --git a/clients/filesystem-hadoop3-runtime/build.gradle.kts b/clients/filesystem-hadoop3-runtime/build.gradle.kts index bc086e0271f..93b936e72ce 100644 --- a/clients/filesystem-hadoop3-runtime/build.gradle.kts +++ b/clients/filesystem-hadoop3-runtime/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar diff --git a/clients/filesystem-hadoop3/build.gradle.kts b/clients/filesystem-hadoop3/build.gradle.kts index fc6a2d8078e..39d20339882 100644 --- a/clients/filesystem-hadoop3/build.gradle.kts +++ b/clients/filesystem-hadoop3/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ plugins { diff --git a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/FilesetContext.java b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/FilesetContext.java index ed1aa5b1397..8aa54dc6835 100644 --- a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/FilesetContext.java +++ b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/FilesetContext.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.filesystem.hadoop; diff --git a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index 92f0152cac0..f2c1ccb33e6 100644 --- a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.filesystem.hadoop; @@ -333,8 +347,7 @@ NameIdentifier extractIdentifier(URI virtualUri) { "URI %s doesn't contains valid identifier", virtualPath); - return NameIdentifier.ofFileset( - metalakeName, matcher.group(1), matcher.group(2), matcher.group(3)); + return NameIdentifier.of(metalakeName, matcher.group(1), matcher.group(2), matcher.group(3)); } private FilesetContext getFilesetContext(Path virtualPath) { @@ -379,7 +392,9 @@ private Pair constructNewFilesetPair(NameIdentifier identif private Fileset loadFileset(NameIdentifier identifier) { Catalog catalog = client.loadCatalog(identifier.namespace().level(1)); - return catalog.asFilesetCatalog().loadFileset(identifier); + return catalog + .asFilesetCatalog() + .loadFileset(NameIdentifier.of(identifier.namespace().level(2), identifier.name())); } @Override diff --git a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java index 323a1eb09cd..e7d2997340a 100644 --- a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java +++ b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.filesystem.hadoop; @@ -60,7 +74,7 @@ class GravitinoVirtualFileSystemConfiguration { "fs.gravitino.fileset.cache.evictionMillsAfterAccess"; public static final long FS_GRAVITINO_FILESET_CACHE_EVICTION_MILLS_AFTER_ACCESS_DEFAULT = - 1000L * 60 * 5; + 1000L * 60 * 60; private GravitinoVirtualFileSystemConfiguration() {} } diff --git a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/Gvfs.java b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/Gvfs.java index 356ae2ea414..a6cfc88e04f 100644 --- a/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/Gvfs.java +++ b/clients/filesystem-hadoop3/src/main/java/com/datastrato/gravitino/filesystem/hadoop/Gvfs.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.filesystem.hadoop; diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/FileSystemTestUtils.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/FileSystemTestUtils.java index 3c61adc1ce0..ea226655975 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/FileSystemTestUtils.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/FileSystemTestUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.filesystem.hadoop; diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoMockServerBase.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoMockServerBase.java index 12d934a7716..b8faa91ff64 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoMockServerBase.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/GravitinoMockServerBase.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.filesystem.hadoop; diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/KdcServerBase.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/KdcServerBase.java index 82e3aa25fac..4066799a669 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/KdcServerBase.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/KdcServerBase.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.filesystem.hadoop; diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/Oauth2MockServerBase.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/Oauth2MockServerBase.java index b0e929a9051..ec5ecabc1ed 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/Oauth2MockServerBase.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/Oauth2MockServerBase.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.filesystem.hadoop; diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java index c77981ad078..dfe4894214c 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.filesystem.hadoop; @@ -504,50 +518,41 @@ public void testExtractIdentifier() throws IOException, URISyntaxException { (GravitinoVirtualFileSystem) managedFilesetPath.getFileSystem(conf)) { NameIdentifier identifier = fs.extractIdentifier(new URI("gvfs://fileset/catalog1/schema1/fileset1")); - assertEquals( - NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier); + assertEquals(NameIdentifier.of(metalakeName, "catalog1", "schema1", "fileset1"), identifier); NameIdentifier identifier2 = fs.extractIdentifier(new URI("gvfs://fileset/catalog1/schema1/fileset1/")); - assertEquals( - NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier2); + assertEquals(NameIdentifier.of(metalakeName, "catalog1", "schema1", "fileset1"), identifier2); NameIdentifier identifier3 = fs.extractIdentifier(new URI("gvfs://fileset/catalog1/schema1/fileset1/files")); - assertEquals( - NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier3); + assertEquals(NameIdentifier.of(metalakeName, "catalog1", "schema1", "fileset1"), identifier3); NameIdentifier identifier4 = fs.extractIdentifier(new URI("gvfs://fileset/catalog1/schema1/fileset1/dir/dir")); - assertEquals( - NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier4); + assertEquals(NameIdentifier.of(metalakeName, "catalog1", "schema1", "fileset1"), identifier4); NameIdentifier identifier5 = fs.extractIdentifier(new URI("gvfs://fileset/catalog1/schema1/fileset1/dir/dir/")); - assertEquals( - NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier5); + assertEquals(NameIdentifier.of(metalakeName, "catalog1", "schema1", "fileset1"), identifier5); NameIdentifier identifier6 = fs.extractIdentifier(new URI("/catalog1/schema1/fileset1")); - assertEquals( - NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier6); + assertEquals(NameIdentifier.of(metalakeName, "catalog1", "schema1", "fileset1"), identifier6); NameIdentifier identifier7 = fs.extractIdentifier(new URI("/catalog1/schema1/fileset1/")); - assertEquals( - NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier7); + assertEquals(NameIdentifier.of(metalakeName, "catalog1", "schema1", "fileset1"), identifier7); NameIdentifier identifier8 = fs.extractIdentifier(new URI("/catalog1/schema1/fileset1/dir")); - assertEquals( - NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier8); + assertEquals(NameIdentifier.of(metalakeName, "catalog1", "schema1", "fileset1"), identifier8); NameIdentifier identifier9 = fs.extractIdentifier(new URI("/catalog1/schema1/fileset1/dir/dir/")); - assertEquals( - NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier9); + assertEquals(NameIdentifier.of(metalakeName, "catalog1", "schema1", "fileset1"), identifier9); NameIdentifier identifier10 = fs.extractIdentifier(new URI("/catalog1/schema1/fileset1/dir/dir")); assertEquals( - NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier10); + NameIdentifier.of(metalakeName, "catalog1", "schema1", "fileset1"), identifier10); StringBuilder longUri = new StringBuilder("gvfs://fileset/catalog1/schema1/fileset1"); for (int i = 0; i < 1500; i++) { @@ -555,11 +560,11 @@ public void testExtractIdentifier() throws IOException, URISyntaxException { } NameIdentifier identifier11 = fs.extractIdentifier(new URI(longUri.toString())); assertEquals( - NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier11); + NameIdentifier.of(metalakeName, "catalog1", "schema1", "fileset1"), identifier11); NameIdentifier identifier12 = fs.extractIdentifier(new URI(longUri.delete(0, 14).toString())); assertEquals( - NameIdentifier.ofFileset(metalakeName, "catalog1", "schema1", "fileset1"), identifier12); + NameIdentifier.of(metalakeName, "catalog1", "schema1", "fileset1"), identifier12); assertThrows( IllegalArgumentException.class, diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestKerberosClient.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestKerberosClient.java index dcb9febabdf..6dacbe03393 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestKerberosClient.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestKerberosClient.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.filesystem.hadoop; diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestOauth2Client.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestOauth2Client.java index c5952d1b3f6..650d2bdf3ff 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestOauth2Client.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestOauth2Client.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.filesystem.hadoop; diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestSimpleClient.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestSimpleClient.java index 12c15c80e32..4a3dc90e893 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestSimpleClient.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestSimpleClient.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.filesystem.hadoop; diff --git a/common/build.gradle.kts b/common/build.gradle.kts index 47a5226076f..1f341257cfd 100644 --- a/common/build.gradle.kts +++ b/common/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ import java.text.SimpleDateFormat @@ -72,8 +86,22 @@ fun writeProjectPropertiesFile() { propertiesFile.writer().use { writer -> writer.write( "#\n" + - "# Copyright 2023 Datastrato Pvt Ltd.\n" + - "# This software is licensed under the Apache License version 2.\n" + + "# Licensed to the Apache Software Foundation (ASF) under one\n" + + "# or more contributor license agreements. See the NOTICE file\n" + + "# distributed with this work for additional information\n" + + "# regarding copyright ownership. The ASF licenses this file\n" + + "# to you under the Apache License, Version 2.0 (the\n" + + "# \"License\"); you may not use this file except in compliance\n" + + "# with the License. You may obtain a copy of the License at\n" + + "#\n" + + "# http://www.apache.org/licenses/LICENSE-2.0\n" + + "#\n" + + "# Unless required by applicable law or agreed to in writing,\n" + + "# software distributed under the License is distributed on an\n" + + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n" + + "# KIND, either express or implied. See the License for the\n" + + "# specific language governing permissions and limitations\n" + + "# under the License.\n" + "#\n" ) writer.write("project.version=$projectVersion\n") diff --git a/common/src/main/java/com/datastrato/gravitino/Version.java b/common/src/main/java/com/datastrato/gravitino/Version.java index 4ed6472d912..32c098c6a8f 100644 --- a/common/src/main/java/com/datastrato/gravitino/Version.java +++ b/common/src/main/java/com/datastrato/gravitino/Version.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java b/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java index 4b105093c92..c1af9e40112 100644 --- a/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java +++ b/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.auth; diff --git a/common/src/main/java/com/datastrato/gravitino/auth/AuthenticatorType.java b/common/src/main/java/com/datastrato/gravitino/auth/AuthenticatorType.java index dd18f805a17..a46abf2b9c8 100644 --- a/common/src/main/java/com/datastrato/gravitino/auth/AuthenticatorType.java +++ b/common/src/main/java/com/datastrato/gravitino/auth/AuthenticatorType.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.auth; diff --git a/common/src/main/java/com/datastrato/gravitino/auth/SignatureAlgorithmFamilyType.java b/common/src/main/java/com/datastrato/gravitino/auth/SignatureAlgorithmFamilyType.java index bc2a9c4861b..0f40e9d1d23 100644 --- a/common/src/main/java/com/datastrato/gravitino/auth/SignatureAlgorithmFamilyType.java +++ b/common/src/main/java/com/datastrato/gravitino/auth/SignatureAlgorithmFamilyType.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.auth; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/AuditDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/AuditDTO.java index ef226eb40bc..f58d9370bb5 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/AuditDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/AuditDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/CatalogDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/CatalogDTO.java index 5025a09f08b..e1bbd3c1e70 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/CatalogDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/CatalogDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/MetalakeDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/MetalakeDTO.java index 80b2156fb12..97053f5f62f 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/MetalakeDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/MetalakeDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/SchemaDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/SchemaDTO.java index c07a9abdbf1..f616b7ca4d5 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/SchemaDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/SchemaDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/VersionDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/VersionDTO.java index a8c4ff1f8bb..51b96b40072 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/VersionDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/VersionDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/authorization/GroupDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/authorization/GroupDTO.java index 2c4720bc9d4..1d442a9539d 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/authorization/GroupDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/authorization/GroupDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.authorization; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/authorization/PrivilegeDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/authorization/PrivilegeDTO.java index b9e65a06fe2..9ff820e441d 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/authorization/PrivilegeDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/authorization/PrivilegeDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.authorization; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/authorization/RoleDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/authorization/RoleDTO.java index ec7e793f21a..a04885258b7 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/authorization/RoleDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/authorization/RoleDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.authorization; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/authorization/SecurableObjectDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/authorization/SecurableObjectDTO.java index 5105b1e68f0..eaa1b89b2fb 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/authorization/SecurableObjectDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/authorization/SecurableObjectDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.authorization; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/authorization/UserDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/authorization/UserDTO.java index f76a912c900..68248bfd82a 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/authorization/UserDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/authorization/UserDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.authorization; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/file/FilesetDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/file/FilesetDTO.java index d285327281f..b2877afb096 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/file/FilesetDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/file/FilesetDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.file; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/messaging/TopicDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/messaging/TopicDTO.java index f060f3d98f1..49f58feb892 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/messaging/TopicDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/messaging/TopicDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.messaging; @@ -90,13 +104,30 @@ public int hashCode() { return Objects.hash(name, comment, properties, audit); } + @Override + public String toString() { + return "TopicDTO{" + + "name='" + + name + + '\'' + + ", comment='" + + comment + + '\'' + + ", properties=" + + properties + + ", audit=" + + audit + + '}'; + } + /** A builder for constructing a Topic DTO. */ public static class Builder { - private final TopicDTO topic; + private String name; + private String comment; + private Map properties; + private AuditDTO audit; - private Builder() { - topic = new TopicDTO(); - } + private Builder() {} /** * Sets the name of the topic. @@ -105,7 +136,7 @@ private Builder() { * @return The builder instance. */ public Builder withName(String name) { - topic.name = name; + this.name = name; return this; } @@ -116,7 +147,7 @@ public Builder withName(String name) { * @return The builder instance. */ public Builder withComment(String comment) { - topic.comment = comment; + this.comment = comment; return this; } @@ -127,7 +158,7 @@ public Builder withComment(String comment) { * @return The builder instance. */ public Builder withProperties(Map properties) { - topic.properties = properties; + this.properties = properties; return this; } @@ -138,13 +169,13 @@ public Builder withProperties(Map properties) { * @return The builder instance. */ public Builder withAudit(AuditDTO audit) { - topic.audit = audit; + this.audit = audit; return this; } /** @return The constructed Topic DTO. */ public TopicDTO build() { - return topic; + return new TopicDTO(name, comment, properties, audit); } } } diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/ColumnDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/ColumnDTO.java index 76544dc692a..49c98adc27c 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/ColumnDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/ColumnDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/DistributionDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/DistributionDTO.java index 9e1e4e11fd5..d3687849799 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/DistributionDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/DistributionDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/PartitionUtils.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/PartitionUtils.java index 66e56b89411..f7827596a30 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/PartitionUtils.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/PartitionUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/SortOrderDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/SortOrderDTO.java index 1d9eb002276..4fd047d2e03 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/SortOrderDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/SortOrderDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java index 84dd54b9917..3cc9d4c3aa9 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel; @@ -58,7 +72,7 @@ private TableDTO() {} * @param properties The properties associated with the table. * @param audit The audit information for the table. * @param partitioning The partitioning of the table. - * @param indexes Teh indexes of the table. + * @param indexes The indexes of the table. */ private TableDTO( String name, diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/FieldReferenceDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/FieldReferenceDTO.java index 3aa53324b18..0d6a4976312 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/FieldReferenceDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/FieldReferenceDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.expressions; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/FuncExpressionDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/FuncExpressionDTO.java index e3e78b56f08..4e556a9a4a3 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/FuncExpressionDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/FuncExpressionDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.expressions; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/FunctionArg.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/FunctionArg.java index 5d32a42aab9..340a9767a5f 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/FunctionArg.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/FunctionArg.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.expressions; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/LiteralDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/LiteralDTO.java index 19c92f2e87f..15418f9e1f7 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/LiteralDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/LiteralDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.expressions; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/UnparsedExpressionDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/UnparsedExpressionDTO.java index a631e702c18..29533250bf8 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/UnparsedExpressionDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/expressions/UnparsedExpressionDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.expressions; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/IndexDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/IndexDTO.java index 2766b16f8bc..bf29c3c0aee 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/IndexDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/IndexDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.indexes; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/BucketPartitioningDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/BucketPartitioningDTO.java index 3efc48152fd..32780aba528 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/BucketPartitioningDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/BucketPartitioningDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitioning; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/DayPartitioningDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/DayPartitioningDTO.java index ebd712cd6f9..21311d2e3bd 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/DayPartitioningDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/DayPartitioningDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitioning; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/FunctionPartitioningDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/FunctionPartitioningDTO.java index d0094d77a6c..7ae9b9b36d1 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/FunctionPartitioningDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/FunctionPartitioningDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitioning; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/HourPartitioningDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/HourPartitioningDTO.java index 7c9c76fa168..abe6494595c 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/HourPartitioningDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/HourPartitioningDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitioning; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/IdentityPartitioningDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/IdentityPartitioningDTO.java index 60b83423b81..c22c937b2fb 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/IdentityPartitioningDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/IdentityPartitioningDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitioning; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/ListPartitioningDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/ListPartitioningDTO.java index 77f1f789c2d..84dd51c35b0 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/ListPartitioningDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/ListPartitioningDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitioning; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/MonthPartitioningDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/MonthPartitioningDTO.java index a1f603dcd75..4db4c8727fc 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/MonthPartitioningDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/MonthPartitioningDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitioning; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/Partitioning.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/Partitioning.java index c53e99d584f..6a2c3a5573d 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/Partitioning.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/Partitioning.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitioning; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/RangePartitioningDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/RangePartitioningDTO.java index 22ede95f044..2444df9a21a 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/RangePartitioningDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/RangePartitioningDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitioning; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/TruncatePartitioningDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/TruncatePartitioningDTO.java index 6d8aca27ff1..b7f669db638 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/TruncatePartitioningDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/TruncatePartitioningDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitioning; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/YearPartitioningDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/YearPartitioningDTO.java index e0678a7d718..c80fb4f5937 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/YearPartitioningDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitioning/YearPartitioningDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitioning; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/IdentityPartitionDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/IdentityPartitionDTO.java index 09265dbc24e..ff7c8b50a19 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/IdentityPartitionDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/IdentityPartitionDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitions; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/ListPartitionDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/ListPartitionDTO.java index 0237b823587..21487066843 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/ListPartitionDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/ListPartitionDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitions; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/PartitionDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/PartitionDTO.java index b97f79eac12..99b514596e1 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/PartitionDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/PartitionDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitions; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/RangePartitionDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/RangePartitionDTO.java index ec0948de6f1..d8f1fd4d064 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/RangePartitionDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/partitions/RangePartitionDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel.partitions; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/AddPartitionsRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/AddPartitionsRequest.java index a64e0a7db65..c6561cd2c14 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/AddPartitionsRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/AddPartitionsRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/CatalogCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/CatalogCreateRequest.java index d85a444a64f..8d31d4dfa0c 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/CatalogCreateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/CatalogCreateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/CatalogUpdateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/CatalogUpdateRequest.java index 9d9c0d5214e..49234baf703 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/CatalogUpdateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/CatalogUpdateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/CatalogUpdatesRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/CatalogUpdatesRequest.java index 9a26792a601..d930152ef33 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/CatalogUpdatesRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/CatalogUpdatesRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetCreateRequest.java index 3e914830b28..91e41bd6743 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetCreateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetCreateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetUpdateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetUpdateRequest.java index 11d6c699146..38a908fba1e 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetUpdateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetUpdateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetUpdatesRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetUpdatesRequest.java index bcb578ee6d1..f2045c84d9f 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetUpdatesRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetUpdatesRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/GroupAddRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/GroupAddRequest.java index 5c41fdaefe6..774f9308656 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/GroupAddRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/GroupAddRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/MetalakeCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/MetalakeCreateRequest.java index d01d0af3f04..de127a87ff8 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/MetalakeCreateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/MetalakeCreateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/MetalakeUpdateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/MetalakeUpdateRequest.java index 0041e25fe30..cf49af4d5cd 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/MetalakeUpdateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/MetalakeUpdateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/MetalakeUpdatesRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/MetalakeUpdatesRequest.java index 149acd91db2..bb448f95678 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/MetalakeUpdatesRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/MetalakeUpdatesRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleCreateRequest.java index f8ed4d1b1b1..08ac059e113 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleCreateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleCreateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleGrantRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleGrantRequest.java index 2bd6880ee21..0b1b34c6df2 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleGrantRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleGrantRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleRevokeRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleRevokeRequest.java index 02f3a3ac31b..72a7f93fe1a 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleRevokeRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/RoleRevokeRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/SchemaCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/SchemaCreateRequest.java index a7cc10c533d..e1856570375 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/SchemaCreateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/SchemaCreateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/SchemaUpdateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/SchemaUpdateRequest.java index e7da41182fe..195ab52123d 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/SchemaUpdateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/SchemaUpdateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/SchemaUpdatesRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/SchemaUpdatesRequest.java index 834741a3433..821c5fb0739 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/SchemaUpdatesRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/SchemaUpdatesRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TableCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TableCreateRequest.java index 3b7e53a06a1..265e01c6786 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TableCreateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TableCreateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TableUpdateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TableUpdateRequest.java index e7856e12470..a31d7c76cd7 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TableUpdateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TableUpdateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TableUpdatesRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TableUpdatesRequest.java index 6125f8471e5..215c6db5b95 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TableUpdatesRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TableUpdatesRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TopicCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TopicCreateRequest.java index df4ca507125..a1749258d34 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TopicCreateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TopicCreateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TopicUpdateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TopicUpdateRequest.java index 1d1f9918b1b..d7c52525ac8 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TopicUpdateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TopicUpdateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TopicUpdatesRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TopicUpdatesRequest.java index e7ea29edf30..a092587499e 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TopicUpdatesRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TopicUpdatesRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/UserAddRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/UserAddRequest.java index 886f4d2f309..1c258438bfb 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/UserAddRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/UserAddRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/BaseResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/BaseResponse.java index 0a18fb93be3..04c610b29d2 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/BaseResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/BaseResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/CatalogListResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/CatalogListResponse.java index 377f6445407..eeedbbae5fd 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/CatalogListResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/CatalogListResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/CatalogResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/CatalogResponse.java index 470671afe48..84f25167175 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/CatalogResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/CatalogResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/DeleteResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/DeleteResponse.java index a524520f256..582a0b99cf3 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/DeleteResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/DeleteResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/DropResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/DropResponse.java index 71fbf5e6de6..79d1bb8d55f 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/DropResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/DropResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/EntityListResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/EntityListResponse.java index 5926e7ae45e..2653d9c3cff 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/EntityListResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/EntityListResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/ErrorConstants.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/ErrorConstants.java index ac4eab28e1b..2e0d913b1db 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/ErrorConstants.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/ErrorConstants.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/ErrorResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/ErrorResponse.java index 4a07982aae0..0f88aee4e89 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/ErrorResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/ErrorResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/FilesetResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/FilesetResponse.java index a6b8910ae61..c7e1003513f 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/FilesetResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/FilesetResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/GroupResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/GroupResponse.java index 100f2fa13bb..59a7a0bedca 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/GroupResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/GroupResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/MetalakeListResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/MetalakeListResponse.java index 7e4e2ccd155..72077eb4ee4 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/MetalakeListResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/MetalakeListResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/MetalakeResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/MetalakeResponse.java index bc0750a2d65..9fd718e98f0 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/MetalakeResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/MetalakeResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/OAuth2ErrorResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/OAuth2ErrorResponse.java index bb26a930f41..f19894ec17e 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/OAuth2ErrorResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/OAuth2ErrorResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/PartitionListResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/PartitionListResponse.java index 13236550fec..911fb31db51 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/PartitionListResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/PartitionListResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/PartitionNameListResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/PartitionNameListResponse.java index cdf3863e695..6641357dc44 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/PartitionNameListResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/PartitionNameListResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/PartitionResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/PartitionResponse.java index 4578fd2972c..8ea73f2bff0 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/PartitionResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/PartitionResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/RemoveResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/RemoveResponse.java index 84bba97fb2f..74252e653ae 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/RemoveResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/RemoveResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/RoleResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/RoleResponse.java index a948207259f..ce809595494 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/RoleResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/RoleResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/SchemaResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/SchemaResponse.java index 444f5966540..2a529de2e1d 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/SchemaResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/SchemaResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/TableResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/TableResponse.java index 74e208138a0..95a0474f10b 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/TableResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/TableResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/TopicResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/TopicResponse.java index d51561abb7b..33b9eb450ab 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/TopicResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/TopicResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/UserResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/UserResponse.java index 277be31d6a9..47662c707d8 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/UserResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/UserResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/VersionResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/VersionResponse.java index 0539f27911b..d6d0412c0d0 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/VersionResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/VersionResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java index c140112d59d..18791effc11 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.util; @@ -316,13 +330,19 @@ public static Partitioning toDTO(Transform transform) { } else if (transform instanceof Transforms.ListTransform) { Transforms.ListTransform listTransform = (Transforms.ListTransform) transform; - return ListPartitioningDTO.of( - listTransform.fieldNames(), (ListPartitionDTO[]) toDTOs(listTransform.assignments())); + ListPartitionDTO[] assignments = + Arrays.stream(listTransform.assignments()) + .map(DTOConverters::toDTO) + .toArray(ListPartitionDTO[]::new); + return ListPartitioningDTO.of(listTransform.fieldNames(), assignments); } else if (transform instanceof Transforms.RangeTransform) { Transforms.RangeTransform rangeTransform = (Transforms.RangeTransform) transform; - return RangePartitioningDTO.of( - rangeTransform.fieldName(), (RangePartitionDTO[]) toDTOs(rangeTransform.assignments())); + RangePartitionDTO[] assignments = + Arrays.stream(rangeTransform.assignments()) + .map(DTOConverters::toDTO) + .toArray(RangePartitionDTO[]::new); + return RangePartitioningDTO.of(rangeTransform.fieldName(), assignments); } else if (transform instanceof Transforms.ApplyTransform) { return FunctionPartitioningDTO.of(transform.name(), toFunctionArg(transform.arguments())); @@ -699,22 +719,33 @@ public static Partition fromDTO(PartitionDTO partitionDTO) { switch (partitionDTO.type()) { case IDENTITY: IdentityPartitionDTO identityPartitionDTO = (IdentityPartitionDTO) partitionDTO; + Literal[] values = + Arrays.stream(identityPartitionDTO.values()) + .map(DTOConverters::fromFunctionArg) + .toArray(Literal[]::new); return Partitions.identity( identityPartitionDTO.name(), identityPartitionDTO.fieldNames(), - identityPartitionDTO.values(), + values, identityPartitionDTO.properties()); case RANGE: RangePartitionDTO rangePartitionDTO = (RangePartitionDTO) partitionDTO; return Partitions.range( rangePartitionDTO.name(), - rangePartitionDTO.upper(), - rangePartitionDTO.lower(), + (Literal) fromFunctionArg(rangePartitionDTO.upper()), + (Literal) fromFunctionArg(rangePartitionDTO.lower()), rangePartitionDTO.properties()); case LIST: ListPartitionDTO listPartitionDTO = (ListPartitionDTO) partitionDTO; - return Partitions.list( - listPartitionDTO.name(), listPartitionDTO.lists(), listPartitionDTO.properties()); + Literal[][] lists = + Arrays.stream(listPartitionDTO.lists()) + .map( + list -> + Arrays.stream(list) + .map(DTOConverters::fromFunctionArg) + .toArray(Literal[]::new)) + .toArray(Literal[][]::new); + return Partitions.list(listPartitionDTO.name(), lists, listPartitionDTO.properties()); default: throw new IllegalArgumentException("Unsupported partition type: " + partitionDTO.type()); } diff --git a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java index 9799eb50889..7e2e72d100a 100644 --- a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java +++ b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.json; @@ -129,9 +143,13 @@ public class JsonUtils { ImmutableList.of( Types.BooleanType.get(), Types.ByteType.get(), + Types.ByteType.unsigned(), Types.ShortType.get(), + Types.ShortType.unsigned(), Types.IntegerType.get(), + Types.IntegerType.unsigned(), Types.LongType.get(), + Types.LongType.unsigned(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), diff --git a/common/src/main/java/com/datastrato/gravitino/rest/RESTRequest.java b/common/src/main/java/com/datastrato/gravitino/rest/RESTRequest.java index e4078357fcd..2ce5514b4c9 100644 --- a/common/src/main/java/com/datastrato/gravitino/rest/RESTRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/rest/RESTRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rest; diff --git a/common/src/main/java/com/datastrato/gravitino/rest/RESTResponse.java b/common/src/main/java/com/datastrato/gravitino/rest/RESTResponse.java index 1b93263425a..6d6840c7ed6 100644 --- a/common/src/main/java/com/datastrato/gravitino/rest/RESTResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/rest/RESTResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rest; diff --git a/common/src/main/java/com/datastrato/gravitino/utils/MapUtils.java b/common/src/main/java/com/datastrato/gravitino/utils/MapUtils.java index 2a235fe13e3..55c6f0594da 100644 --- a/common/src/main/java/com/datastrato/gravitino/utils/MapUtils.java +++ b/common/src/main/java/com/datastrato/gravitino/utils/MapUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.utils; diff --git a/common/src/main/java/com/datastrato/gravitino/utils/RandomNameUtils.java b/common/src/main/java/com/datastrato/gravitino/utils/RandomNameUtils.java index 23752eca9e0..8b3d012f626 100644 --- a/common/src/main/java/com/datastrato/gravitino/utils/RandomNameUtils.java +++ b/common/src/main/java/com/datastrato/gravitino/utils/RandomNameUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.utils; diff --git a/common/src/main/resources/project.properties b/common/src/main/resources/project.properties new file mode 100644 index 00000000000..3cb0b0b8245 --- /dev/null +++ b/common/src/main/resources/project.properties @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +project.version=0.5.1-SNAPSHOT +compile.date=30/05/2024 20:03:36 +git.commit.id=3dc0376e5eaf2fece68b7e0dcc885f22fa03acf6 diff --git a/common/src/test/java/com/datastrato/gravitino/dto/rel/TestDistributionDTO.java b/common/src/test/java/com/datastrato/gravitino/dto/rel/TestDistributionDTO.java index b9abecea54d..61644191799 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/rel/TestDistributionDTO.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/rel/TestDistributionDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel; diff --git a/common/src/test/java/com/datastrato/gravitino/dto/rel/TestSortOrderDTO.java b/common/src/test/java/com/datastrato/gravitino/dto/rel/TestSortOrderDTO.java index b5bbf2af3f6..2b1f25deaf2 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/rel/TestSortOrderDTO.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/rel/TestSortOrderDTO.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.rel; diff --git a/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTableUpdatesRequest.java b/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTableUpdatesRequest.java index 0ff4c4b9821..1bc9a6f4800 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTableUpdatesRequest.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTableUpdatesRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java b/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java index 16fb335a352..f068213419d 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/test/java/com/datastrato/gravitino/dto/util/TestDTOConverters.java b/common/src/test/java/com/datastrato/gravitino/dto/util/TestDTOConverters.java index 51db69f3024..62f22254685 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/util/TestDTOConverters.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/util/TestDTOConverters.java @@ -1,17 +1,36 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.util; import com.datastrato.gravitino.dto.rel.expressions.LiteralDTO; +import com.datastrato.gravitino.dto.rel.partitioning.ListPartitioningDTO; +import com.datastrato.gravitino.dto.rel.partitioning.RangePartitioningDTO; import com.datastrato.gravitino.dto.rel.partitions.IdentityPartitionDTO; import com.datastrato.gravitino.dto.rel.partitions.ListPartitionDTO; import com.datastrato.gravitino.dto.rel.partitions.PartitionDTO; import com.datastrato.gravitino.dto.rel.partitions.RangePartitionDTO; import com.datastrato.gravitino.rel.expressions.literals.Literal; import com.datastrato.gravitino.rel.expressions.literals.Literals; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.expressions.transforms.Transforms; +import com.datastrato.gravitino.rel.partitions.IdentityPartition; import com.datastrato.gravitino.rel.partitions.ListPartition; import com.datastrato.gravitino.rel.partitions.Partition; import com.datastrato.gravitino.rel.partitions.Partitions; @@ -37,6 +56,10 @@ void testIdentityPartitionDTOConvert() { LiteralDTO.builder().withDataType(Types.StringType.get()).withValue("us").build(); String[][] fieldNames = {field1, field2}; LiteralDTO[] values = {literal1, literal2}; + Literal[] expectedValues = { + (Literal) DTOConverters.fromFunctionArg(literal1), + (Literal) DTOConverters.fromFunctionArg(literal2) + }; Map properties = Collections.singletonMap("key", "value"); PartitionDTO identityPartitionDTO = @@ -47,14 +70,13 @@ void testIdentityPartitionDTOConvert() { .withProperties(properties) .build(); // when - com.datastrato.gravitino.rel.partitions.IdentityPartition identityPartition = - (com.datastrato.gravitino.rel.partitions.IdentityPartition) - DTOConverters.fromDTO(identityPartitionDTO); + IdentityPartition identityPartition = + (IdentityPartition) DTOConverters.fromDTO(identityPartitionDTO); // then Assertions.assertTrue(Arrays.equals(fieldNames, identityPartition.fieldNames())); Assertions.assertEquals("IdentityPartition", identityPartition.name()); - Assertions.assertTrue(Arrays.equals(values, identityPartition.values())); + Assertions.assertTrue(Arrays.equals(expectedValues, identityPartition.values())); Assertions.assertEquals(properties, identityPartition.properties()); } @@ -66,6 +88,8 @@ void testRangePartitionDTOConvert() { LiteralDTO.builder().withDataType(Types.DateType.get()).withValue("2008-08-08").build(); LiteralDTO upper = LiteralDTO.builder().withDataType(Types.StringType.get()).withValue("us").build(); + Literal expectedLower = (Literal) DTOConverters.fromFunctionArg(lower); + Literal expectedUpper = (Literal) DTOConverters.fromFunctionArg(upper); Map properties = Collections.singletonMap("key", "value"); PartitionDTO rangePartitionDTO = @@ -80,8 +104,8 @@ void testRangePartitionDTOConvert() { // then Assertions.assertEquals("RangePartition", rangePartition.name()); - Assertions.assertEquals(lower, rangePartition.lower()); - Assertions.assertEquals(upper, rangePartition.upper()); + Assertions.assertEquals(expectedLower, rangePartition.lower()); + Assertions.assertEquals(expectedUpper, rangePartition.upper()); Assertions.assertEquals(properties, rangePartition.properties()); } @@ -96,6 +120,10 @@ void testListPartitionDTOConvert() { Map properties = Collections.singletonMap("key", "value"); LiteralDTO[][] literalDTOS = {new LiteralDTO[] {literal1}, new LiteralDTO[] {literal2}}; + Literal[][] expectedValues = { + new Literal[] {(Literal) DTOConverters.fromFunctionArg(literal1)}, + new Literal[] {(Literal) DTOConverters.fromFunctionArg(literal2)} + }; ListPartitionDTO listPartitionDTO = ListPartitionDTO.builder() .withName("ListPartition") @@ -108,7 +136,7 @@ void testListPartitionDTOConvert() { // then Assertions.assertEquals("ListPartition", listPartition.name()); - Assertions.assertTrue(Arrays.equals(literalDTOS, listPartition.lists())); + Assertions.assertTrue(Arrays.deepEquals(expectedValues, listPartition.lists())); Assertions.assertEquals(properties, listPartition.properties()); } @@ -181,4 +209,41 @@ void testListPartitionConvert() { Assertions.assertEquals(values.length, listPartitionDTO.lists().length); Assertions.assertEquals(properties, listPartitionDTO.properties()); } + + @Test + void testPartitionDTOConvert() { + Literal lower = Literals.stringLiteral("2008-08-08"); + Literal upper = Literals.stringLiteral("us"); + Map properties = Collections.singletonMap("key", "value"); + RangePartition rangePartition = Partitions.range("range", upper, lower, properties); + Transform rangeTransform = + Transforms.range(new String[] {"col1"}, new RangePartition[] {rangePartition}); + + RangePartitioningDTO rangePartitioning = + (RangePartitioningDTO) DTOConverters.toDTO(rangeTransform); + String[] rangePartitionFieldName = rangePartitioning.fieldName(); + RangePartitionDTO[] rangePartitionAssignments = rangePartitioning.assignments(); + + Assertions.assertEquals("col1", rangePartitionFieldName[0]); + Assertions.assertEquals(lower.dataType(), rangePartitionAssignments[0].lower().dataType()); + Assertions.assertEquals("2008-08-08", rangePartitionAssignments[0].lower().value()); + Assertions.assertEquals(upper.dataType(), rangePartitionAssignments[0].upper().dataType()); + Assertions.assertEquals("us", rangePartitionAssignments[0].upper().value()); + Assertions.assertEquals(properties, rangePartitionAssignments[0].properties()); + + Literal value = Literals.stringLiteral(Types.StringType.get().simpleString()); + Literal[][] values = {new Literal[] {value}}; + ListPartition listPartition = Partitions.list("list", values, properties); + Transform listTransform = + Transforms.list(new String[][] {{"col2"}}, new ListPartition[] {listPartition}); + ListPartitioningDTO listPartitioning = (ListPartitioningDTO) DTOConverters.toDTO(listTransform); + String[][] listPartitionFieldNames = listPartitioning.fieldNames(); + ListPartitionDTO[] listPartitionAssignments = listPartitioning.assignments(); + + Assertions.assertEquals("col2", listPartitionFieldNames[0][0]); + Assertions.assertEquals(value.dataType(), listPartitionAssignments[0].lists()[0][0].dataType()); + Assertions.assertEquals( + Types.StringType.get().simpleString(), listPartitionAssignments[0].lists()[0][0].value()); + Assertions.assertEquals(properties, listPartitionAssignments[0].properties()); + } } diff --git a/common/src/test/java/com/datastrato/gravitino/json/TestDTOJsonSerDe.java b/common/src/test/java/com/datastrato/gravitino/json/TestDTOJsonSerDe.java index f6b51d7fccf..07907dca752 100644 --- a/common/src/test/java/com/datastrato/gravitino/json/TestDTOJsonSerDe.java +++ b/common/src/test/java/com/datastrato/gravitino/json/TestDTOJsonSerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.json; diff --git a/common/src/test/java/com/datastrato/gravitino/json/TestJsonUtils.java b/common/src/test/java/com/datastrato/gravitino/json/TestJsonUtils.java index 4439858a84a..bd5c436901a 100644 --- a/common/src/test/java/com/datastrato/gravitino/json/TestJsonUtils.java +++ b/common/src/test/java/com/datastrato/gravitino/json/TestJsonUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.json; diff --git a/common/src/test/java/com/datastrato/gravitino/json/TestRequestJsonSerDe.java b/common/src/test/java/com/datastrato/gravitino/json/TestRequestJsonSerDe.java index 90ce26d28e3..1f4753a1225 100644 --- a/common/src/test/java/com/datastrato/gravitino/json/TestRequestJsonSerDe.java +++ b/common/src/test/java/com/datastrato/gravitino/json/TestRequestJsonSerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.json; diff --git a/common/src/test/java/com/datastrato/gravitino/json/TestResponseJsonSerDe.java b/common/src/test/java/com/datastrato/gravitino/json/TestResponseJsonSerDe.java index b9942ebfd24..f417dd9a7df 100644 --- a/common/src/test/java/com/datastrato/gravitino/json/TestResponseJsonSerDe.java +++ b/common/src/test/java/com/datastrato/gravitino/json/TestResponseJsonSerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.json; diff --git a/common/src/test/java/com/datastrato/gravitino/rest/TestRESTUtils.java b/common/src/test/java/com/datastrato/gravitino/rest/TestRESTUtils.java index 5fcc648149f..259826da4c0 100644 --- a/common/src/test/java/com/datastrato/gravitino/rest/TestRESTUtils.java +++ b/common/src/test/java/com/datastrato/gravitino/rest/TestRESTUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.rest; diff --git a/common/src/test/java/com/datastrato/gravitino/utils/TestMapUtils.java b/common/src/test/java/com/datastrato/gravitino/utils/TestMapUtils.java index c7fb8b3be3b..a036ac44a09 100644 --- a/common/src/test/java/com/datastrato/gravitino/utils/TestMapUtils.java +++ b/common/src/test/java/com/datastrato/gravitino/utils/TestMapUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.utils; diff --git a/common/src/test/java/resources/log4j2.properties b/common/src/test/java/resources/log4j2.properties index b9e706603a0..f3155e4c00b 100644 --- a/common/src/test/java/resources/log4j2.properties +++ b/common/src/test/java/resources/log4j2.properties @@ -1,6 +1,20 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # Set to debug or trace if log4j initialization is failing diff --git a/conf/gravitino-env.sh.template b/conf/gravitino-env.sh.template index b37b8ef7100..656ca176b2b 100644 --- a/conf/gravitino-env.sh.template +++ b/conf/gravitino-env.sh.template @@ -1,6 +1,20 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # Debug Gravitino server diff --git a/conf/gravitino.conf.template b/conf/gravitino.conf.template index 5a6cf006380..22fe9241553 100644 --- a/conf/gravitino.conf.template +++ b/conf/gravitino.conf.template @@ -1,6 +1,20 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # # THE CONFIGURATION FOR Gravitino SERVER @@ -27,15 +41,19 @@ gravitino.server.webserver.requestHeaderSize = 131072 gravitino.server.webserver.responseHeaderSize = 131072 # THE CONFIGURATION FOR Gravitino ENTITY STORE -# The entity store to use, kv or relational -gravitino.entity.store = kv +# The entity store to use, we only supports relational +gravitino.entity.store = relational +# The backend for the entity store, we only supports JDBC +gravitino.entity.store.relational = JDBCBackend -# The RocksDB entity store -gravitino.entity.store.kv = RocksDBKvBackend -# The storage path for RocksDB storage implementation, it supports both absolute and relative path, -# If the value is a relative path, the final path is "${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}", default value -# is "${GRAVITINO_HOME}/data/rocksdb", please uncomment and change it in your production environment. -# gravitino.entity.store.kv.rocksdbPath = data/rocksdb +# The JDBC URL for the entity store +gravitino.entity.store.relational.jdbcUrl = jdbc:h2 +# The JDBC driver class name +gravitino.entity.store.relational.jdbcDriver = org.h2.Driver +# The JDBC user name +gravitino.entity.store.relational.jdbcUser = gravitino +# The JDBC password +gravitino.entity.store.relational.jdbcPassword = gravitino # THE CONFIGURATION FOR Gravitino CATALOG # The interval in milliseconds to evict the catalog cache diff --git a/conf/log4j2.properties.template b/conf/log4j2.properties.template index e1954be7d1d..9931e9c1b20 100644 --- a/conf/log4j2.properties.template +++ b/conf/log4j2.properties.template @@ -1,6 +1,20 @@ # -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # status = warn diff --git a/core/build.gradle.kts b/core/build.gradle.kts index a07b29ae5b7..f2b70744e96 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ plugins { `maven-publish` @@ -20,6 +34,7 @@ dependencies { implementation(libs.commons.io) implementation(libs.commons.lang3) implementation(libs.guava) + implementation(libs.h2db) implementation(libs.mybatis) implementation(libs.protobuf.java.util) { exclude("com.google.guava", "guava") @@ -36,10 +51,18 @@ dependencies { testCompileOnly(libs.lombok) testImplementation(libs.awaitility) - testImplementation(libs.h2db) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.mockito.core) testRuntimeOnly(libs.junit.jupiter.engine) } + +tasks.test { + val testMode = project.properties["testMode"] as? String ?: "embedded" + if (testMode == "embedded") { + environment("GRAVITINO_HOME", project.rootDir.path) + } else { + environment("GRAVITINO_HOME", project.rootDir.path + "/distribution/package") + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/Config.java b/core/src/main/java/com/datastrato/gravitino/Config.java index d5712c9fec7..91046c77977 100644 --- a/core/src/main/java/com/datastrato/gravitino/Config.java +++ b/core/src/main/java/com/datastrato/gravitino/Config.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index 26619f03464..5ee2232caf3 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; @@ -35,6 +49,9 @@ private Configs() {} public static final String ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY = "gravitino.entity.store.relational.jdbcPassword"; + public static final String ENTITY_RELATIONAL_JDBC_BACKEND_STORAGE_PATH_KEY = + "gravitino.entity.store.relational.storagePath"; + public static final String ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY = "gravitino.entity.store.kv.rocksdbPath"; @@ -66,6 +83,17 @@ private Configs() {} public static final String DEFAULT_KV_ROCKSDB_BACKEND_PATH = String.join(File.separator, System.getenv("GRAVITINO_HOME"), "data", "rocksdb"); + public static final String DEFAULT_RELATIONAL_JDBC_BACKEND_PATH = + String.join(File.separator, System.getenv("GRAVITINO_HOME"), "data", "jdbc"); + + public static final String DEFAULT_RELATIONAL_JDBC_BACKEND_URL = "jdbc:h2"; + + public static final String DEFAULT_RELATIONAL_JDBC_BACKEND_DRIVER = "org.h2.Driver"; + + public static final String DEFAULT_RELATIONAL_JDBC_BACKEND_USERNAME = "gravitino"; + + public static final String DEFAULT_RELATIONAL_JDBC_BACKEND_PASSWORD = "gravitino"; + public static final int GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT = 100; public static final long MAX_NODE_IN_MEMORY = 100000L; @@ -78,7 +106,7 @@ private Configs() {} .doc("Which storage implementation to use") .version(ConfigConstants.VERSION_0_1_0) .stringConf() - .createWithDefault(DEFAULT_ENTITY_STORE); + .createWithDefault(RELATIONAL_ENTITY_STORE); public static final ConfigEntry ENTITY_KV_STORE = new ConfigBuilder(ENTITY_KV_STORE_KEY) @@ -101,7 +129,7 @@ private Configs() {} .version(ConfigConstants.VERSION_0_5_0) .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) - .create(); + .createWithDefault(DEFAULT_RELATIONAL_JDBC_BACKEND_URL); public static final ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER = new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY) @@ -109,7 +137,7 @@ private Configs() {} .version(ConfigConstants.VERSION_0_5_0) .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) - .create(); + .createWithDefault(DEFAULT_RELATIONAL_JDBC_BACKEND_DRIVER); public static final ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_USER = new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY) @@ -117,17 +145,27 @@ private Configs() {} .version(ConfigConstants.VERSION_0_5_0) .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) - .create(); + .createWithDefault(DEFAULT_RELATIONAL_JDBC_BACKEND_USERNAME); public static final ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD = new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY) .doc("Password of `JDBCBackend`") .version(ConfigConstants.VERSION_0_5_0) .stringConf() - .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) - .create(); + .createWithDefault(DEFAULT_RELATIONAL_JDBC_BACKEND_PASSWORD); + + public static final ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_PATH = + new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_STORAGE_PATH_KEY) + .doc( + "The storage path for JDBC storage implementation. It supports both absolute and" + + " relative path, if the value is a relative path, the final path is " + + "`${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is " + + "`${GRAVITINO_HOME}/data/jdbc`") + .version(ConfigConstants.VERSION_0_6_0) + .stringConf() + .createWithDefault(DEFAULT_RELATIONAL_JDBC_BACKEND_PATH); - public static final ConfigEntry ENTRY_KV_ROCKSDB_BACKEND_PATH = + public static final ConfigEntry ENTITY_KV_ROCKSDB_BACKEND_PATH = new ConfigBuilder(ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY) .doc( "The storage path for RocksDB storage implementation. It supports both absolute and" diff --git a/core/src/main/java/com/datastrato/gravitino/Entity.java b/core/src/main/java/com/datastrato/gravitino/Entity.java index d8d6ab03cd7..932264650b8 100644 --- a/core/src/main/java/com/datastrato/gravitino/Entity.java +++ b/core/src/main/java/com/datastrato/gravitino/Entity.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; @@ -23,9 +37,6 @@ public interface Entity extends Serializable { /** The system reserved catalog name. */ String SYSTEM_CATALOG_RESERVED_NAME = "system"; - /** The securable object reserved entity name. */ - String SECURABLE_ENTITY_RESERVED_NAME = "*"; - /** The authorization catalog name in the system metalake. */ String AUTHORIZATION_CATALOG_NAME = "authorization"; @@ -34,12 +45,28 @@ public interface Entity extends Serializable { /** The group schema name in the system catalog. */ String GROUP_SCHEMA_NAME = "group"; + /** The role schema name in the system catalog. */ String ROLE_SCHEMA_NAME = "role"; /** The admin schema name in the authorization catalog of the system metalake. */ String ADMIN_SCHEMA_NAME = "admin"; + /** The tag schema name in the system catalog. */ + String TAG_SCHEMA_NAME = "tag"; + + /** + * All metalakes are a virtual entity. It represents all the metalakes. We don't store it. We use + * a specific type to represent its entity type. + */ + String ALL_METALAKES_ENTITY_TYPE = "ROOT"; + + /** + * All metalakes are a virtual entity. It represents all the metalakes. We don't store it. We use + * a specific id to represent its entity id. + */ + long ALL_METALAKES_ENTITY_ID = 0; + /** Enumeration defining the types of entities in the Gravitino framework. */ @Getter enum EntityType { @@ -53,6 +80,7 @@ enum EntityType { USER("us", 7), GROUP("gr", 8), ROLE("ro", 9), + TAG("ta", 10), AUDIT("au", 65534); diff --git a/core/src/main/java/com/datastrato/gravitino/EntityAlreadyExistsException.java b/core/src/main/java/com/datastrato/gravitino/EntityAlreadyExistsException.java index c29fd3a95e1..a119d5ed558 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntityAlreadyExistsException.java +++ b/core/src/main/java/com/datastrato/gravitino/EntityAlreadyExistsException.java @@ -1,31 +1,54 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + /** * Exception class indicating that an entity already exists. This exception is thrown when an * attempt is made to create an entity that already exists within the Gravitino framework. */ -public class EntityAlreadyExistsException extends RuntimeException { +public class EntityAlreadyExistsException extends GravitinoRuntimeException { /** * Constructs an EntityAlreadyExistsException. * - * @param message The detail message explaining the exception. + * @param message the detail message. + * @param args the arguments to the message. */ - public EntityAlreadyExistsException(String message) { - super(message); + @FormatMethod + public EntityAlreadyExistsException(@FormatString String message, Object... args) { + super(message, args); } /** * Constructs an EntityAlreadyExistsException. * - * @param message The detail message explaining the exception. - * @param cause The cause of the exception. + * @param cause the cause. + * @param message the detail message. + * @param args the arguments to the message. */ - public EntityAlreadyExistsException(String message, Throwable cause) { - super(message, cause); + @FormatMethod + public EntityAlreadyExistsException( + Throwable cause, @FormatString String message, Object... args) { + super(cause, message, args); } } diff --git a/core/src/main/java/com/datastrato/gravitino/EntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/EntitySerDe.java index 103ed34ed83..33dbff37d45 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/EntitySerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java b/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java index fc59fe77357..409805b0e02 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java +++ b/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/core/src/main/java/com/datastrato/gravitino/EntityStore.java b/core/src/main/java/com/datastrato/gravitino/EntityStore.java index e2e5a55fdcf..b52a8ec4d2f 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/EntityStore.java @@ -1,11 +1,24 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; import com.datastrato.gravitino.Entity.EntityType; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.utils.Executable; import java.io.Closeable; @@ -110,11 +123,11 @@ void put(E e, boolean overwritten) * @return E the updated entity * @throws IOException if the store operation fails * @throws NoSuchEntityException if the entity does not exist - * @throws AlreadyExistsException if the updated entity already existed. + * @throws EntityAlreadyExistsException if the updated entity already existed. */ E update( NameIdentifier ident, Class type, EntityType entityType, Function updater) - throws IOException, NoSuchEntityException, AlreadyExistsException; + throws IOException, NoSuchEntityException, EntityAlreadyExistsException; /** * Get the entity from the underlying storage. diff --git a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java index dfc083e7e64..2625812dbf7 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java +++ b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/core/src/main/java/com/datastrato/gravitino/Field.java b/core/src/main/java/com/datastrato/gravitino/Field.java index ca179b0b3c5..b88ac7d0eb1 100644 --- a/core/src/main/java/com/datastrato/gravitino/Field.java +++ b/core/src/main/java/com/datastrato/gravitino/Field.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java index a05bb3662ee..8bdbd3354fa 100644 --- a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java +++ b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; @@ -40,7 +54,7 @@ import com.datastrato.gravitino.metrics.source.JVMMetricsSource; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.storage.RandomIdGenerator; -import com.google.common.annotations.VisibleForTesting; +import com.datastrato.gravitino.tag.TagManager; import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,8 +95,11 @@ public class GravitinoEnv { private MetricsSystem metricsSystem; private LockManager lockManager; + private EventListenerManager eventListenerManager; + private TagManager tagManager; + private GravitinoEnv() {} private static class InstanceHolder { @@ -98,40 +115,6 @@ public static GravitinoEnv getInstance() { return InstanceHolder.INSTANCE; } - /** - * This method is used for testing purposes only to set the lock manager for test in package - * `com.datastrato.gravitino.server.web.rest`, as tree lock depends on the lock manager and we did - * not mock the lock manager in the test, so we need to set the lock manager for test. - * - * @param lockManager The lock manager to be set. - */ - @VisibleForTesting - public void setLockManager(LockManager lockManager) { - this.lockManager = lockManager; - } - - /** - * This method is used for testing purposes only to set the access manager for test in package - * `com.datastrato.gravitino.server.web.rest` and `com.datastrato.gravitino.authorization`. - * - * @param accessControlManager The access control manager to be set. - */ - @VisibleForTesting - public void setAccessControlManager(AccessControlManager accessControlManager) { - this.accessControlManager = accessControlManager; - } - - /** - * This method is used for testing purposes only to set the entity store for test in package - * `com.datastrato.gravitino.authorization`. - * - * @param entityStore The entity store to be set. - */ - @VisibleForTesting - public void setEntityStore(EntityStore entityStore) { - this.entityStore = entityStore; - } - /** * Initialize the Gravitino environment. * @@ -211,6 +194,10 @@ public void initialize(Config config) { // Tree lock this.lockManager = new LockManager(config); + + // Tag manager + this.tagManager = new TagManager(idGenerator, entityStore); + LOG.info("Gravitino Environment is initialized."); } @@ -322,6 +309,15 @@ public AccessControlManager accessControlManager() { return accessControlManager; } + /** + * Get the TagManager associated with the Gravitino environment. + * + * @return The TagManager instance. + */ + public TagManager tagManager() { + return tagManager; + } + public void start() { auxServiceManager.serviceStart(); metricsSystem.start(); diff --git a/core/src/main/java/com/datastrato/gravitino/HasIdentifier.java b/core/src/main/java/com/datastrato/gravitino/HasIdentifier.java index 9d8e5e2e5ae..cb12f6873fc 100644 --- a/core/src/main/java/com/datastrato/gravitino/HasIdentifier.java +++ b/core/src/main/java/com/datastrato/gravitino/HasIdentifier.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/core/src/main/java/com/datastrato/gravitino/StringIdentifier.java b/core/src/main/java/com/datastrato/gravitino/StringIdentifier.java index 2c929e81bc9..46af042e54a 100644 --- a/core/src/main/java/com/datastrato/gravitino/StringIdentifier.java +++ b/core/src/main/java/com/datastrato/gravitino/StringIdentifier.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/core/src/main/java/com/datastrato/gravitino/UnsupportedEntityTypeException.java b/core/src/main/java/com/datastrato/gravitino/UnsupportedEntityTypeException.java index 8f0eb31fef6..6998de5ed2c 100644 --- a/core/src/main/java/com/datastrato/gravitino/UnsupportedEntityTypeException.java +++ b/core/src/main/java/com/datastrato/gravitino/UnsupportedEntityTypeException.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/core/src/main/java/com/datastrato/gravitino/UserPrincipal.java b/core/src/main/java/com/datastrato/gravitino/UserPrincipal.java index a869c1de369..8d3ea8654f3 100644 --- a/core/src/main/java/com/datastrato/gravitino/UserPrincipal.java +++ b/core/src/main/java/com/datastrato/gravitino/UserPrincipal.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino; diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java index d8024b924ff..231d38af876 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; @@ -249,17 +263,20 @@ public boolean isMetalakeAdmin(String user) { * @param metalake The Metalake of the Role. * @param role The name of the Role. * @param properties The properties of the Role. - * @param securableObject The securable object of the Role. + * @param securableObjects The securable objects of the Role. * @return The created Role instance. * @throws RoleAlreadyExistsException If a Role with the same name already exists. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If creating the Role encounters storage issues. */ public Role createRole( - String metalake, String role, Map properties, SecurableObject securableObject) + String metalake, + String role, + Map properties, + List securableObjects) throws RoleAlreadyExistsException, NoSuchMetalakeException { return doWithNonAdminLock( - () -> roleManager.createRole(metalake, role, properties, securableObject)); + () -> roleManager.createRole(metalake, role, properties, securableObjects)); } /** diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AdminManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/AdminManager.java index 74ee9054027..6dabd909961 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AdminManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AdminManager.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AuthorizationUtils.java b/core/src/main/java/com/datastrato/gravitino/authorization/AuthorizationUtils.java index ad9db0d0b5a..776500b46ba 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AuthorizationUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AuthorizationUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; @@ -29,7 +43,7 @@ static void checkMetalakeExists(String metalake) throws NoSuchMetalakeException try { EntityStore store = GravitinoEnv.getInstance().entityStore(); - NameIdentifier metalakeIdent = NameIdentifier.ofMetalake(metalake); + NameIdentifier metalakeIdent = NameIdentifier.of(metalake); if (!store.exists(metalakeIdent, Entity.EntityType.METALAKE)) { LOG.warn("Metalake {} does not exist", metalakeIdent); throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalakeIdent); diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java index c8e6060ab52..f30f418771b 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java index 654aab9a023..96fc7170250 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; @@ -72,7 +86,10 @@ class RoleManager { } RoleEntity createRole( - String metalake, String role, Map properties, SecurableObject securableObject) + String metalake, + String role, + Map properties, + List securableObjects) throws RoleAlreadyExistsException { AuthorizationUtils.checkMetalakeExists(metalake); RoleEntity roleEntity = @@ -80,7 +97,7 @@ RoleEntity createRole( .withId(idGenerator.nextId()) .withName(role) .withProperties(properties) - .withSecurableObject(securableObject) + .withSecurableObjects(securableObjects) .withNamespace(AuthorizationUtils.ofRoleNamespace(metalake)) .withAuditInfo( AuditInfo.builder() diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java index f353996e9fc..c21b060a304 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.authorization; diff --git a/core/src/main/java/com/datastrato/gravitino/auxiliary/AuxiliaryServiceManager.java b/core/src/main/java/com/datastrato/gravitino/auxiliary/AuxiliaryServiceManager.java index 1ed3f7221b1..98b3468e319 100644 --- a/core/src/main/java/com/datastrato/gravitino/auxiliary/AuxiliaryServiceManager.java +++ b/core/src/main/java/com/datastrato/gravitino/auxiliary/AuxiliaryServiceManager.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.auxiliary; diff --git a/core/src/main/java/com/datastrato/gravitino/auxiliary/GravitinoAuxiliaryService.java b/core/src/main/java/com/datastrato/gravitino/auxiliary/GravitinoAuxiliaryService.java index e08daaa1972..dc1700eaa80 100644 --- a/core/src/main/java/com/datastrato/gravitino/auxiliary/GravitinoAuxiliaryService.java +++ b/core/src/main/java/com/datastrato/gravitino/auxiliary/GravitinoAuxiliaryService.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.auxiliary; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java b/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java index 8d108c8f49d..6005cce9771 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CapabilityHelpers.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; @@ -24,6 +38,11 @@ import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.indexes.Index; import com.datastrato.gravitino.rel.indexes.Indexes; +import com.datastrato.gravitino.rel.partitions.IdentityPartition; +import com.datastrato.gravitino.rel.partitions.ListPartition; +import com.datastrato.gravitino.rel.partitions.Partition; +import com.datastrato.gravitino.rel.partitions.Partitions; +import com.datastrato.gravitino.rel.partitions.RangePartition; import com.google.common.base.Preconditions; import java.util.Arrays; @@ -110,6 +129,39 @@ public static Namespace applyCaseSensitive( return namespace; } + public static Partition[] applyCaseSensitive(Partition[] partitions, Capability capabilities) { + return Arrays.stream(partitions) + .map(p -> applyCaseSensitive(p, capabilities)) + .toArray(Partition[]::new); + } + + public static Partition applyCaseSensitive(Partition partition, Capability capabilities) { + String newName = + capabilities.caseSensitiveOnName(Capability.Scope.PARTITION).supported() + ? partition.name() + : partition.name().toLowerCase(); + if (partition instanceof IdentityPartition) { + IdentityPartition identityPartition = (IdentityPartition) partition; + return Partitions.identity( + newName, + identityPartition.fieldNames(), + identityPartition.values(), + identityPartition.properties()); + + } else if (partition instanceof ListPartition) { + ListPartition listPartition = (ListPartition) partition; + return Partitions.list(newName, listPartition.lists(), listPartition.properties()); + + } else if (partition instanceof RangePartition) { + RangePartition rangePartition = (RangePartition) partition; + return Partitions.range( + newName, rangePartition.upper(), rangePartition.lower(), rangePartition.properties()); + + } else { + throw new IllegalArgumentException("Unknown partition type: " + partition.getClass()); + } + } + public static Transform[] applyCapabilities(Transform[] transforms, Capability capabilities) { return Arrays.stream(transforms) .map(t -> applyCapabilities(t, capabilities)) @@ -198,12 +250,22 @@ private static Transform applyCapabilities(Transform transform, Capability capab applyCapabilities(truncateTransform.fieldName(), capabilities)); } else if (transform instanceof Transforms.ListTransform) { + Transforms.ListTransform listTransform = (Transforms.ListTransform) transform; + ListPartition[] assignments = + Arrays.stream(listTransform.assignments()) + .map(l -> applyCaseSensitive(l, capabilities)) + .toArray(ListPartition[]::new); return Transforms.list( - applyCapabilities(((Transforms.ListTransform) transform).fieldNames(), capabilities)); + applyCapabilities(listTransform.fieldNames(), capabilities), assignments); } else if (transform instanceof Transforms.RangeTransform) { + Transforms.RangeTransform rangeTransform = (Transforms.RangeTransform) transform; + RangePartition[] assignments = + Arrays.stream(rangeTransform.assignments()) + .map(r -> applyCaseSensitive(r, capabilities)) + .toArray(RangePartition[]::new); return Transforms.range( - applyCapabilities(((Transforms.RangeTransform) transform).fieldName(), capabilities)); + applyCapabilities(rangeTransform.fieldName(), capabilities), assignments); } else if (transform instanceof Transforms.ApplyTransform) { return Transforms.apply( diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogDispatcher.java index 38b4fda4962..dd547ac9cd1 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogDispatcher.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 87e36036eca..65189359c56 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; @@ -506,7 +520,7 @@ public boolean dropCatalog(NameIdentifier ident) { // Kafka catalog needs to cascade drop the default schema List schemas = store.list( - Namespace.ofSchema(ident.namespace().level(0), ident.name()), + Namespace.of(ident.namespace().level(0), ident.name()), SchemaEntity.class, EntityType.SCHEMA); // If there is only one schema, it must be the default schema, because we don't allow to diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogNormalizeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogNormalizeDispatcher.java index 41b3ed042ca..74d1feb1371 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogNormalizeDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogNormalizeDispatcher.java @@ -1,14 +1,28 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; -import static com.datastrato.gravitino.Entity.SECURABLE_ENTITY_RESERVED_NAME; import static com.datastrato.gravitino.Entity.SYSTEM_CATALOG_RESERVED_NAME; import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.CatalogChange; +import com.datastrato.gravitino.MetadataObjects; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.CatalogAlreadyExistsException; @@ -21,7 +35,7 @@ public class CatalogNormalizeDispatcher implements CatalogDispatcher { private static final Set RESERVED_WORDS = - ImmutableSet.of(SECURABLE_ENTITY_RESERVED_NAME, SYSTEM_CATALOG_RESERVED_NAME); + ImmutableSet.of(MetadataObjects.METADATA_OBJECT_RESERVED_NAME, SYSTEM_CATALOG_RESERVED_NAME); /** * Regular expression explanation: * diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedFileset.java b/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedFileset.java index ef45172118d..12e63e24fe3 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedFileset.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedFileset.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedSchema.java b/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedSchema.java index 02e45e6eb81..aec17013714 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedSchema.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedSchema.java @@ -1,11 +1,26 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; import com.datastrato.gravitino.Audit; import com.datastrato.gravitino.Schema; +import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.SchemaEntity; import java.util.Map; @@ -19,14 +34,23 @@ public final class EntityCombinedSchema implements Schema { private final Schema schema; + private final SchemaEntity schemaEntity; // Sets of properties that should be hidden from the user. private Set hiddenProperties; + // Field "imported" is used to indicate whether the entity has been imported to Gravitino + // managed storage backend. If "imported" is true, it means that storage backend have stored + // the correct entity. Otherwise, we should import the external entity to the storage backend. + // This is used for tag/access control related purposes, only the imported entities have the + // unique id, and based on this id, we can label and control the access to the entities. + private boolean imported; + private EntityCombinedSchema(Schema schema, SchemaEntity schemaEntity) { this.schema = schema; this.schemaEntity = schemaEntity; + this.imported = false; } public static EntityCombinedSchema of(Schema schema, SchemaEntity schemaEntity) { @@ -42,6 +66,11 @@ public EntityCombinedSchema withHiddenPropertiesSet(Set hiddenProperties return this; } + public EntityCombinedSchema withImported(boolean imported) { + this.imported = imported; + return this; + } + @Override public String name() { return schema.name(); @@ -73,4 +102,12 @@ public Audit auditInfo() { ? schema.auditInfo() : mergedAudit.merge(schemaEntity.auditInfo(), true /* overwrite */); } + + public boolean imported() { + return imported; + } + + StringIdentifier stringIdentifier() { + return StringIdentifier.fromProperties(schema.properties()); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedTable.java b/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedTable.java index 593508f9e6e..5b2553660ba 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedTable.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedTable.java @@ -1,10 +1,25 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; import com.datastrato.gravitino.Audit; +import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.TableEntity; import com.datastrato.gravitino.rel.Column; @@ -31,9 +46,17 @@ public final class EntityCombinedTable implements Table { // Sets of properties that should be hidden from the user. private Set hiddenProperties; + // Field "imported" is used to indicate whether the entity has been imported to Gravitino + // managed storage backend. If "imported" is true, it means that storage backend have stored + // the correct entity. Otherwise, we should import the external entity to the storage backend. + // This is used for tag/access control related purposes, only the imported entities have the + // unique id, and based on this id, we can label and control the access to the entities. + private boolean imported; + private EntityCombinedTable(Table table, TableEntity tableEntity) { this.table = table; this.tableEntity = tableEntity; + this.imported = false; } public static EntityCombinedTable of(Table table, TableEntity tableEntity) { @@ -49,6 +72,11 @@ public EntityCombinedTable withHiddenPropertiesSet(Set hiddenProperties) return this; } + public EntityCombinedTable withImported(boolean imported) { + this.imported = imported; + return this; + } + @Override public String name() { return table.name(); @@ -96,6 +124,10 @@ public Index[] index() { return table.index(); } + public boolean imported() { + return imported; + } + @Override public Audit auditInfo() { AuditInfo mergedAudit = @@ -110,4 +142,8 @@ public Audit auditInfo() { ? table.auditInfo() : mergedAudit.merge(tableEntity.auditInfo(), true /* overwrite */); } + + StringIdentifier stringIdentifier() { + return StringIdentifier.fromProperties(table.properties()); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedTopic.java b/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedTopic.java index b6fed19ec32..7fb17fff2e0 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedTopic.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/EntityCombinedTopic.java @@ -1,10 +1,25 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; import com.datastrato.gravitino.Audit; +import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.messaging.Topic; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.TopicEntity; @@ -24,9 +39,17 @@ public class EntityCombinedTopic implements Topic { // Sets of properties that should be hidden from the user. private Set hiddenProperties; + // Field "imported" is used to indicate whether the entity has been imported to Gravitino + // managed storage backend. If "imported" is true, it means that storage backend have stored + // the correct entity. Otherwise, we should import the external entity to the storage backend. + // This is used for tag/access control related purposes, only the imported entities have the + // unique id, and based on this id, we can label and control the access to the entities. + private boolean imported; + private EntityCombinedTopic(Topic topic, TopicEntity topicEntity) { this.topic = topic; this.topicEntity = topicEntity; + this.imported = false; } public static EntityCombinedTopic of(Topic topic, TopicEntity topicEntity) { @@ -42,6 +65,11 @@ public EntityCombinedTopic withHiddenPropertiesSet(Set hiddenProperties) return this; } + public EntityCombinedTopic withImported(boolean imported) { + this.imported = imported; + return this; + } + @Override public String name() { return topic.name(); @@ -73,4 +101,12 @@ public Audit auditInfo() { ? topic.auditInfo() : mergedAudit.merge(topicEntity.auditInfo(), true /* overwrite */); } + + public boolean imported() { + return imported; + } + + StringIdentifier stringIdentifier() { + return StringIdentifier.fromProperties(topic.properties()); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetDispatcher.java index f2d0daf280d..287ffb8b095 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetDispatcher.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetNormalizeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetNormalizeDispatcher.java index 6dd66565cbe..7aa8707df20 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetNormalizeDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetNormalizeDispatcher.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java index 30c012b1f5f..fe9fa154ffe 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/FilesetOperationDispatcher.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/OperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/OperationDispatcher.java index a24a258d398..febbbbf0b6f 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/OperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/OperationDispatcher.java @@ -1,11 +1,26 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; import static com.datastrato.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter; +import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; @@ -44,7 +59,7 @@ public abstract class OperationDispatcher { protected final EntityStore store; - final IdGenerator idGenerator; + protected final IdGenerator idGenerator; /** * Creates a new CatalogOperationDispatcher instance. @@ -60,7 +75,21 @@ public OperationDispatcher( this.idGenerator = idGenerator; } - R doWithTable( + protected Capability getCatalogCapability(NameIdentifier ident) { + return doWithCatalog( + getCatalogIdentifier(ident), + CatalogManager.CatalogWrapper::capabilities, + IllegalArgumentException.class); + } + + protected Capability getCatalogCapability(Namespace namespace) { + return doWithCatalog( + getCatalogIdentifier(NameIdentifier.of(namespace.levels())), + CatalogManager.CatalogWrapper::capabilities, + IllegalArgumentException.class); + } + + protected R doWithTable( NameIdentifier tableIdent, ThrowableFunction fn, Class ex) throws E { try { @@ -78,7 +107,7 @@ R doWithTable( } } - R doWithCatalog( + protected R doWithCatalog( NameIdentifier ident, ThrowableFunction fn, Class ex) throws E { try { @@ -95,7 +124,7 @@ R doWithCatalog( } } - R doWithCatalog( + protected R doWithCatalog( NameIdentifier ident, ThrowableFunction fn, Class ex1, @@ -118,21 +147,7 @@ R doWithCatalog( } } - Capability getCatalogCapability(NameIdentifier ident) { - return doWithCatalog( - getCatalogIdentifier(ident), - CatalogManager.CatalogWrapper::capabilities, - IllegalArgumentException.class); - } - - Capability getCatalogCapability(Namespace namespace) { - return doWithCatalog( - getCatalogIdentifier(NameIdentifier.of(namespace.levels())), - CatalogManager.CatalogWrapper::capabilities, - IllegalArgumentException.class); - } - - Set getHiddenPropertyNames( + protected Set getHiddenPropertyNames( NameIdentifier catalogIdent, ThrowableFunction provider, Map properties) { @@ -149,7 +164,7 @@ Set getHiddenPropertyNames( IllegalArgumentException.class); } - void validateAlterProperties( + protected void validateAlterProperties( NameIdentifier ident, ThrowableFunction provider, T... changes) { @@ -166,27 +181,6 @@ void validateAlterProperties( IllegalArgumentException.class); } - private Map getPropertiesForSet(T... t) { - Map properties = Maps.newHashMap(); - for (T item : t) { - if (item instanceof TableChange.SetProperty) { - TableChange.SetProperty setProperty = (TableChange.SetProperty) item; - properties.put(setProperty.getProperty(), setProperty.getValue()); - } else if (item instanceof SchemaChange.SetProperty) { - SchemaChange.SetProperty setProperty = (SchemaChange.SetProperty) item; - properties.put(setProperty.getProperty(), setProperty.getValue()); - } else if (item instanceof FilesetChange.SetProperty) { - FilesetChange.SetProperty setProperty = (FilesetChange.SetProperty) item; - properties.put(setProperty.getProperty(), setProperty.getValue()); - } else if (item instanceof TopicChange.SetProperty) { - TopicChange.SetProperty setProperty = (TopicChange.SetProperty) item; - properties.put(setProperty.getProperty(), setProperty.getValue()); - } - } - - return properties; - } - private Map getPropertiesForDelete(T... t) { Map properties = Maps.newHashMap(); for (T item : t) { @@ -208,7 +202,7 @@ private Map getPropertiesForDelete(T... t) { return properties; } - StringIdentifier getStringIdFromProperties(Map properties) { + protected StringIdentifier getStringIdFromProperties(Map properties) { try { StringIdentifier stringId = StringIdentifier.fromProperties(properties); if (stringId == null) { @@ -221,7 +215,7 @@ StringIdentifier getStringIdFromProperties(Map properties) { } } - R operateOnEntity( + protected R operateOnEntity( NameIdentifier ident, ThrowableFunction fn, String opName, long id) { R ret = null; try { @@ -247,7 +241,7 @@ R operateOnEntity( // TODO(xun): Remove this method when we implement a better way to get the catalog identifier // [#257] Add an explicit get catalog functions in NameIdentifier - NameIdentifier getCatalogIdentifier(NameIdentifier ident) { + protected NameIdentifier getCatalogIdentifier(NameIdentifier ident) { NameIdentifier.check( ident.name() != null, "The name variable in the NameIdentifier must have value."); Namespace.check( @@ -272,6 +266,36 @@ boolean isManagedEntity(NameIdentifier catalogIdent, Capability.Scope scope) { IllegalArgumentException.class); } + protected boolean isEntityExist(NameIdentifier ident, Entity.EntityType type) { + try { + return store.exists(ident, type); + } catch (Exception e) { + LOG.error(FormattedErrorMessages.STORE_OP_FAILURE, "exists", ident, e); + throw new RuntimeException("Fail to check if entity is existed", e); + } + } + + private Map getPropertiesForSet(T... t) { + Map properties = Maps.newHashMap(); + for (T item : t) { + if (item instanceof TableChange.SetProperty) { + TableChange.SetProperty setProperty = (TableChange.SetProperty) item; + properties.put(setProperty.getProperty(), setProperty.getValue()); + } else if (item instanceof SchemaChange.SetProperty) { + SchemaChange.SetProperty setProperty = (SchemaChange.SetProperty) item; + properties.put(setProperty.getProperty(), setProperty.getValue()); + } else if (item instanceof FilesetChange.SetProperty) { + FilesetChange.SetProperty setProperty = (FilesetChange.SetProperty) item; + properties.put(setProperty.getProperty(), setProperty.getValue()); + } else if (item instanceof TopicChange.SetProperty) { + TopicChange.SetProperty setProperty = (TopicChange.SetProperty) item; + properties.put(setProperty.getProperty(), setProperty.getValue()); + } + } + + return properties; + } + static final class FormattedErrorMessages { static final String STORE_OP_FAILURE = "Failed to {} entity for {} in " diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/PartitionDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/PartitionDispatcher.java index 13047de9346..e6bb15b095a 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/PartitionDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/PartitionDispatcher.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/PartitionNormalizeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/PartitionNormalizeDispatcher.java index 4d45aa7a0b2..87b8d6ee298 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/PartitionNormalizeDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/PartitionNormalizeDispatcher.java @@ -1,20 +1,31 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; +import static com.datastrato.gravitino.catalog.CapabilityHelpers.applyCaseSensitive; import static com.datastrato.gravitino.catalog.CapabilityHelpers.applyCaseSensitiveOnName; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.connector.capability.Capability; import com.datastrato.gravitino.exceptions.NoSuchPartitionException; import com.datastrato.gravitino.exceptions.PartitionAlreadyExistsException; -import com.datastrato.gravitino.rel.partitions.IdentityPartition; -import com.datastrato.gravitino.rel.partitions.ListPartition; import com.datastrato.gravitino.rel.partitions.Partition; -import com.datastrato.gravitino.rel.partitions.Partitions; -import com.datastrato.gravitino.rel.partitions.RangePartition; import java.util.Arrays; public class PartitionNormalizeDispatcher implements PartitionDispatcher { @@ -29,8 +40,13 @@ public PartitionNormalizeDispatcher(PartitionOperationDispatcher dispatcher) { public String[] listPartitionNames(NameIdentifier tableIdent) { String[] partitionNames = dispatcher.listPartitionNames( - CapabilityHelpers.applyCaseSensitive(tableIdent, Capability.Scope.TABLE, dispatcher)); - return applyCaseSensitive(tableIdent, partitionNames); + applyCaseSensitive(tableIdent, Capability.Scope.TABLE, dispatcher)); + Capability capabilities = dispatcher.getCatalogCapability(tableIdent); + return Arrays.stream(partitionNames) + .map( + partitionName -> + applyCaseSensitiveOnName(Capability.Scope.PARTITION, partitionName, capabilities)) + .toArray(String[]::new); } @Override @@ -38,7 +54,7 @@ public Partition[] listPartitions(NameIdentifier tableIdent) { Partition[] partitions = dispatcher.listPartitions( CapabilityHelpers.applyCaseSensitive(tableIdent, Capability.Scope.TABLE, dispatcher)); - return applyCaseSensitive(tableIdent, partitions); + return applyCaseSensitive(partitions, dispatcher.getCatalogCapability(tableIdent)); } @Override @@ -57,7 +73,7 @@ public Partition addPartition(NameIdentifier tableIdent, Partition partition) throws PartitionAlreadyExistsException { return dispatcher.addPartition( CapabilityHelpers.applyCaseSensitive(tableIdent, Capability.Scope.TABLE, dispatcher), - applyCaseSensitive(tableIdent, partition)); + applyCaseSensitive(partition, dispatcher.getCatalogCapability(tableIdent))); } @Override @@ -80,57 +96,4 @@ public boolean purgePartition(NameIdentifier tableIdent, String partitionName) partitionName, dispatcher.getCatalogCapability(tableIdent))); } - - private String[] applyCaseSensitive(NameIdentifier tableIdent, String[] partitionNames) { - Capability capabilities = dispatcher.getCatalogCapability(tableIdent); - return Arrays.stream(partitionNames) - .map( - partitionName -> - applyCaseSensitiveOnName(Capability.Scope.PARTITION, partitionName, capabilities)) - .toArray(String[]::new); - } - - private Partition[] applyCaseSensitive(NameIdentifier tableIdent, Partition[] partitions) { - boolean caseSensitive = - dispatcher - .getCatalogCapability(tableIdent) - .caseSensitiveOnName(Capability.Scope.PARTITION) - .supported(); - return Arrays.stream(partitions) - .map(partition -> applyCaseSensitive(partition, caseSensitive)) - .toArray(Partition[]::new); - } - - private Partition applyCaseSensitive(NameIdentifier tableIdent, Partition partition) { - boolean caseSensitive = - dispatcher - .getCatalogCapability(tableIdent) - .caseSensitiveOnName(Capability.Scope.PARTITION) - .supported(); - return applyCaseSensitive(partition, caseSensitive); - } - - private Partition applyCaseSensitive(Partition partition, boolean caseSensitive) { - String newName = caseSensitive ? partition.name() : partition.name().toLowerCase(); - if (partition instanceof IdentityPartition) { - IdentityPartition identityPartition = (IdentityPartition) partition; - return Partitions.identity( - newName, - identityPartition.fieldNames(), - identityPartition.values(), - identityPartition.properties()); - - } else if (partition instanceof ListPartition) { - ListPartition listPartition = (ListPartition) partition; - return Partitions.list(newName, listPartition.lists(), listPartition.properties()); - - } else if (partition instanceof RangePartition) { - RangePartition rangePartition = (RangePartition) partition; - return Partitions.range( - newName, rangePartition.upper(), rangePartition.lower(), rangePartition.properties()); - - } else { - throw new IllegalArgumentException("Unknown partition type: " + partition.getClass()); - } - } } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/PartitionOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/PartitionOperationDispatcher.java index 720376218b1..e301725707a 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/PartitionOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/PartitionOperationDispatcher.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/PropertiesMetadataHelpers.java b/core/src/main/java/com/datastrato/gravitino/catalog/PropertiesMetadataHelpers.java index 3da1226351b..b878121cd61 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/PropertiesMetadataHelpers.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/PropertiesMetadataHelpers.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaDispatcher.java index e61838e9556..df3e4054d5c 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaDispatcher.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaNormalizeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaNormalizeDispatcher.java index 9c004989e80..9be0a8fc023 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaNormalizeDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaNormalizeDispatcher.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; @@ -65,10 +79,7 @@ public Schema alterSchema(NameIdentifier ident, SchemaChange... changes) @Override public boolean dropSchema(NameIdentifier ident, boolean cascade) throws NonEmptySchemaException { - // The constraints of the name spec may be more strict than underlying catalog, - // and for compatibility reasons, we only apply case-sensitive capabilities here. - return dispatcher.dropSchema( - applyCaseSensitive(ident, Capability.Scope.SCHEMA, dispatcher), cascade); + return dispatcher.dropSchema(normalizeNameIdentifier(ident), cascade); } private NameIdentifier normalizeNameIdentifier(NameIdentifier ident) { diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java index 1575ded111d..9ab3ccdcd2c 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/SchemaOperationDispatcher.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; @@ -20,6 +34,8 @@ import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NonEmptySchemaException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; +import com.datastrato.gravitino.lock.LockType; +import com.datastrato.gravitino.lock.TreeLockUtils; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.storage.IdGenerator; @@ -159,47 +175,21 @@ public Schema createSchema(NameIdentifier ident, String comment, Map c.doWithSchemaOps(s -> s.loadSchema(ident)), - NoSuchSchemaException.class); - - // If the Schema is maintained by the Gravitino's store, we don't have to load again. - boolean isManagedSchema = isManagedEntity(catalogIdentifier, Capability.Scope.SCHEMA); - if (isManagedSchema) { - return EntityCombinedSchema.of(schema) - .withHiddenPropertiesSet( - getHiddenPropertyNames( - catalogIdentifier, - HasPropertyMetadata::schemaPropertiesMetadata, - schema.properties())); - } + // Load the schema and check if this schema is already imported. + EntityCombinedSchema schema = + TreeLockUtils.doWithTreeLock(ident, LockType.READ, () -> internalLoadSchema(ident)); - StringIdentifier stringId = getStringIdFromProperties(schema.properties()); - // Case 1: The schema is not created by Gravitino. - if (stringId == null) { - return EntityCombinedSchema.of(schema) - .withHiddenPropertiesSet( - getHiddenPropertyNames( - catalogIdentifier, - HasPropertyMetadata::schemaPropertiesMetadata, - schema.properties())); + if (!schema.imported()) { + TreeLockUtils.doWithTreeLock( + NameIdentifier.of(ident.namespace().levels()), + LockType.WRITE, + () -> { + importSchema(ident); + return null; + }); } - SchemaEntity schemaEntity = - operateOnEntity( - ident, - identifier -> store.get(identifier, SCHEMA, SchemaEntity.class), - "GET", - stringId.id()); - return EntityCombinedSchema.of(schema, schemaEntity) - .withHiddenPropertiesSet( - getHiddenPropertyNames( - catalogIdentifier, - HasPropertyMetadata::schemaPropertiesMetadata, - schema.properties())); + return schema; } /** @@ -280,6 +270,7 @@ public Schema alterSchema(NameIdentifier ident, SchemaChange... changes) .build()), "UPDATE", stringId.id()); + return EntityCombinedSchema.of(alteredSchema, updatedSchemaEntity) .withHiddenPropertiesSet( getHiddenPropertyNames( @@ -330,4 +321,106 @@ public boolean dropSchema(NameIdentifier ident, boolean cascade) throws NonEmpty ? droppedFromStore : droppedFromCatalog; } + + private void importSchema(NameIdentifier identifier) { + EntityCombinedSchema schema = internalLoadSchema(identifier); + if (schema.imported()) { + return; + } + + StringIdentifier stringId = null; + try { + stringId = schema.stringIdentifier(); + } catch (IllegalArgumentException ie) { + LOG.warn(FormattedErrorMessages.STRING_ID_PARSE_ERROR, ie.getMessage()); + } + + long uid; + if (stringId != null) { + // If the entity in the store doesn't match the one in the external system, we use the data + // of external system to correct it. + LOG.warn( + "The Schema uid {} existed but still needs to be imported, this could be happened " + + "when Schema is renamed by external systems not controlled by Gravitino. In this case, " + + "we need to overwrite the stored entity to keep consistency.", + stringId); + uid = stringId.id(); + } else { + // If entity doesn't exist, we import the entity from the external system. + uid = idGenerator.nextId(); + } + + SchemaEntity schemaEntity = + SchemaEntity.builder() + .withId(uid) + .withName(identifier.name()) + .withNamespace(identifier.namespace()) + .withAuditInfo( + AuditInfo.builder() + .withCreator(schema.auditInfo().creator()) + .withCreateTime(schema.auditInfo().createTime()) + .withLastModifier(schema.auditInfo().lastModifier()) + .withLastModifiedTime(schema.auditInfo().lastModifiedTime()) + .build()) + .build(); + try { + store.put(schemaEntity, true); + } catch (Exception e) { + LOG.error(FormattedErrorMessages.STORE_OP_FAILURE, "put", identifier, e); + throw new RuntimeException("Fail to import schema entity to the store.", e); + } + } + + private EntityCombinedSchema internalLoadSchema(NameIdentifier ident) { + NameIdentifier catalogIdentifier = getCatalogIdentifier(ident); + Schema schema = + doWithCatalog( + catalogIdentifier, + c -> c.doWithSchemaOps(s -> s.loadSchema(ident)), + NoSuchSchemaException.class); + + // If the Schema is maintained by the entity store, we don't have to import. + boolean isManagedSchema = isManagedEntity(catalogIdentifier, Capability.Scope.SCHEMA); + if (isManagedSchema) { + return EntityCombinedSchema.of(schema) + .withHiddenPropertiesSet( + getHiddenPropertyNames( + catalogIdentifier, + HasPropertyMetadata::schemaPropertiesMetadata, + schema.properties())) + // The meta of managed schema is stored by Gravitino, we don't need to import it. + .withImported(true /* imported */); + } + + StringIdentifier stringId = getStringIdFromProperties(schema.properties()); + // Case 1: The schema is not created by Gravitino or the external system does not support + // storing string identifiers. + if (stringId == null) { + return EntityCombinedSchema.of(schema) + .withHiddenPropertiesSet( + getHiddenPropertyNames( + catalogIdentifier, + HasPropertyMetadata::schemaPropertiesMetadata, + schema.properties())) + // For some catalogs like PG, the identifier information is not stored in the schema's + // metadata, we need to check if this schema is existed in the store, if so we don't + // need to import. + .withImported(isEntityExist(ident, SCHEMA)); + } + + SchemaEntity schemaEntity = + operateOnEntity( + ident, + identifier -> store.get(identifier, SCHEMA, SchemaEntity.class), + "GET", + stringId.id()); + + return EntityCombinedSchema.of(schema, schemaEntity) + .withHiddenPropertiesSet( + getHiddenPropertyNames( + catalogIdentifier, + HasPropertyMetadata::schemaPropertiesMetadata, + schema.properties())) + .withImported(schemaEntity != null); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/SupportsCatalogs.java b/core/src/main/java/com/datastrato/gravitino/catalog/SupportsCatalogs.java index 2e68d1fae80..46d415c2a60 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/SupportsCatalogs.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/SupportsCatalogs.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableDispatcher.java index 7b54ccd5794..b6fb58da064 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TableDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableDispatcher.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableNormalizeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableNormalizeDispatcher.java index ac7da6cbc55..2f83535b745 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TableNormalizeDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableNormalizeDispatcher.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; @@ -82,9 +96,7 @@ public Table alterTable(NameIdentifier ident, TableChange... changes) @Override public boolean dropTable(NameIdentifier ident) { - // The constraints of the name spec may be more strict than underlying catalog, - // and for compatibility reasons, we only apply case-sensitive capabilities here. - return dispatcher.dropTable(applyCaseSensitive(ident, Capability.Scope.TABLE, dispatcher)); + return dispatcher.dropTable(normalizeNameIdentifier(ident)); } @Override diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java index df41cdc2634..df90e65dbe1 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TableOperationDispatcher.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; @@ -10,6 +24,7 @@ import static com.datastrato.gravitino.rel.expressions.transforms.Transforms.EMPTY_TRANSFORM; import com.datastrato.gravitino.EntityStore; +import com.datastrato.gravitino.GravitinoEnv; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; @@ -19,6 +34,8 @@ import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.lock.LockType; +import com.datastrato.gravitino.lock.TreeLockUtils; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.TableEntity; import com.datastrato.gravitino.rel.Column; @@ -79,37 +96,26 @@ public NameIdentifier[] listTables(Namespace namespace) throws NoSuchSchemaExcep */ @Override public Table loadTable(NameIdentifier ident) throws NoSuchTableException { - NameIdentifier catalogIdentifier = getCatalogIdentifier(ident); - Table table = - doWithCatalog( - catalogIdentifier, - c -> c.doWithTableOps(t -> t.loadTable(ident)), - NoSuchTableException.class); + EntityCombinedTable table = + TreeLockUtils.doWithTreeLock(ident, LockType.READ, () -> internalLoadTable(ident)); - StringIdentifier stringId = getStringIdFromProperties(table.properties()); - // Case 1: The table is not created by Gravitino. - if (stringId == null) { - return EntityCombinedTable.of(table) - .withHiddenPropertiesSet( - getHiddenPropertyNames( - catalogIdentifier, - HasPropertyMetadata::tablePropertiesMetadata, - table.properties())); - } + if (!table.imported()) { + // Load the schema to make sure the schema is imported. + SchemaDispatcher schemaDispatcher = GravitinoEnv.getInstance().schemaDispatcher(); + NameIdentifier schemaIdent = NameIdentifier.of(ident.namespace().levels()); + schemaDispatcher.loadSchema(schemaIdent); - TableEntity tableEntity = - operateOnEntity( - ident, - identifier -> store.get(identifier, TABLE, TableEntity.class), - "GET", - stringId.id()); + // Import the table. + TreeLockUtils.doWithTreeLock( + schemaIdent, + LockType.WRITE, + () -> { + importTable(ident); + return null; + }); + } - return EntityCombinedTable.of(table, tableEntity) - .withHiddenPropertiesSet( - getHiddenPropertyNames( - catalogIdentifier, - HasPropertyMetadata::tablePropertiesMetadata, - table.properties())); + return table; } /** @@ -379,4 +385,94 @@ public boolean purgeTable(NameIdentifier ident) throws UnsupportedOperationExcep ? droppedFromStore : droppedFromCatalog; } + + private void importTable(NameIdentifier identifier) { + EntityCombinedTable table = internalLoadTable(identifier); + + if (table.imported()) { + return; + } + + StringIdentifier stringId = null; + try { + stringId = table.stringIdentifier(); + } catch (IllegalArgumentException ie) { + LOG.warn(FormattedErrorMessages.STRING_ID_PARSE_ERROR, ie.getMessage()); + } + + long uid; + if (stringId != null) { + // If the entity in the store doesn't match the external system, we use the data + // of external system to correct it. + LOG.warn( + "The Table uid {} existed but still need to be imported, this could be happened " + + "when Table is renamed by external systems not controlled by Gravitino. In this case, " + + "we need to overwrite the stored entity to keep the consistency.", + stringId); + uid = stringId.id(); + } else { + // If entity doesn't exist, we import the entity from the external system. + uid = idGenerator.nextId(); + } + + TableEntity tableEntity = + TableEntity.builder() + .withId(uid) + .withName(identifier.name()) + .withNamespace(identifier.namespace()) + .withAuditInfo( + AuditInfo.builder() + .withCreator(table.auditInfo().creator()) + .withCreateTime(table.auditInfo().createTime()) + .withLastModifier(table.auditInfo().lastModifier()) + .withLastModifiedTime(table.auditInfo().lastModifiedTime()) + .build()) + .build(); + try { + store.put(tableEntity, true); + } catch (Exception e) { + LOG.error(FormattedErrorMessages.STORE_OP_FAILURE, "put", identifier, e); + throw new RuntimeException("Fail to import the table entity to the store.", e); + } + } + + private EntityCombinedTable internalLoadTable(NameIdentifier ident) { + NameIdentifier catalogIdentifier = getCatalogIdentifier(ident); + Table table = + doWithCatalog( + catalogIdentifier, + c -> c.doWithTableOps(t -> t.loadTable(ident)), + NoSuchTableException.class); + + StringIdentifier stringId = getStringIdFromProperties(table.properties()); + // Case 1: The table is not created by Gravitino or the external system does not support storing + // string identifier. + if (stringId == null) { + return EntityCombinedTable.of(table) + .withHiddenPropertiesSet( + getHiddenPropertyNames( + catalogIdentifier, + HasPropertyMetadata::tablePropertiesMetadata, + table.properties())) + // Some tables don't have properties or are not created by Gravitino, + // we can't use stringIdentifier to judge whether schema is ever imported or not. + // We need to check whether the entity exists. + .withImported(isEntityExist(ident, TABLE)); + } + + TableEntity tableEntity = + operateOnEntity( + ident, + identifier -> store.get(identifier, TABLE, TableEntity.class), + "GET", + stringId.id()); + + return EntityCombinedTable.of(table, tableEntity) + .withHiddenPropertiesSet( + getHiddenPropertyNames( + catalogIdentifier, + HasPropertyMetadata::tablePropertiesMetadata, + table.properties())) + .withImported(tableEntity != null); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TopicDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TopicDispatcher.java index 131a600c621..da2e434612e 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TopicDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TopicDispatcher.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TopicNormalizeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TopicNormalizeDispatcher.java index aa372b623c1..1e46ecc99ff 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TopicNormalizeDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TopicNormalizeDispatcher.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java index 6d52e92d75a..b83a1223df1 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/TopicOperationDispatcher.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.catalog; @@ -9,6 +23,7 @@ import static com.datastrato.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; import com.datastrato.gravitino.EntityStore; +import com.datastrato.gravitino.GravitinoEnv; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; @@ -18,6 +33,8 @@ import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NoSuchTopicException; import com.datastrato.gravitino.exceptions.TopicAlreadyExistsException; +import com.datastrato.gravitino.lock.LockType; +import com.datastrato.gravitino.lock.TreeLockUtils; import com.datastrato.gravitino.messaging.DataLayout; import com.datastrato.gravitino.messaging.Topic; import com.datastrato.gravitino.messaging.TopicChange; @@ -70,36 +87,27 @@ public NameIdentifier[] listTopics(Namespace namespace) throws NoSuchSchemaExcep */ @Override public Topic loadTopic(NameIdentifier ident) throws NoSuchTopicException { - NameIdentifier catalogIdent = getCatalogIdentifier(ident); - Topic topic = - doWithCatalog( - catalogIdent, - c -> c.doWithTopicOps(t -> t.loadTopic(ident)), - NoSuchTopicException.class); + EntityCombinedTopic topic = + TreeLockUtils.doWithTreeLock(ident, LockType.READ, () -> internalLoadTopic(ident)); - StringIdentifier stringId = getStringIdFromProperties(topic.properties()); - // Case 1: The topic is not created by Gravitino. - // Note: for Kafka catalog, stringId will not be null. Because there is no way to store the - // Gravitino - // ID in Kafka, therefor we use the topic ID as the Gravitino ID - if (stringId == null) { - return EntityCombinedTopic.of(topic) - .withHiddenPropertiesSet( - getHiddenPropertyNames( - catalogIdent, HasPropertyMetadata::topicPropertiesMetadata, topic.properties())); - } + if (!topic.imported()) { + // Load the schema to make sure the schema is imported. + // This is not necessary for Kafka catalog. + SchemaDispatcher schemaDispatcher = GravitinoEnv.getInstance().schemaDispatcher(); + NameIdentifier schemaIdent = NameIdentifier.of(ident.namespace().levels()); + schemaDispatcher.loadSchema(schemaIdent); - TopicEntity topicEntity = - operateOnEntity( - ident, - identifier -> store.get(identifier, TOPIC, TopicEntity.class), - "GET", - getStringIdFromProperties(topic.properties()).id()); + // Import the topic + TreeLockUtils.doWithTreeLock( + schemaIdent, + LockType.WRITE, + () -> { + importTopic(ident); + return null; + }); + } - return EntityCombinedTopic.of(topic, topicEntity) - .withHiddenPropertiesSet( - getHiddenPropertyNames( - catalogIdent, HasPropertyMetadata::topicPropertiesMetadata, topic.properties())); + return topic; } /** @@ -282,4 +290,87 @@ public boolean dropTopic(NameIdentifier ident) { ? droppedFromStore : droppedFromCatalog; } + + private void importTopic(NameIdentifier identifier) { + + EntityCombinedTopic topic = internalLoadTopic(identifier); + + if (topic.imported()) { + return; + } + + StringIdentifier stringId = null; + try { + stringId = topic.stringIdentifier(); + } catch (IllegalArgumentException ie) { + LOG.warn(FormattedErrorMessages.STRING_ID_PARSE_ERROR, ie.getMessage()); + } + + long uid; + if (stringId != null) { + // For Kafka topic, the uid is coming from topic UUID, which is always existed. + LOG.warn( + "The Topic uid {} existed but still needs to be imported, this could be happened " + + "when Topic is created externally without leveraging Gravitino. In this " + + "case, we need to store the stored entity to keep consistency.", + stringId); + uid = stringId.id(); + } else { + // This will not be happened for now, since we only support Kafka, and it always has an uid. + uid = idGenerator.nextId(); + } + + TopicEntity topicEntity = + TopicEntity.builder() + .withId(uid) + .withName(topic.name()) + .withComment(topic.comment()) + .withNamespace(identifier.namespace()) + .withAuditInfo( + AuditInfo.builder() + .withCreator(topic.auditInfo().creator()) + .withCreateTime(topic.auditInfo().createTime()) + .withLastModifier(topic.auditInfo().lastModifier()) + .withLastModifiedTime(topic.auditInfo().lastModifiedTime()) + .build()) + .build(); + + try { + store.put(topicEntity, true); + } catch (Exception e) { + LOG.error(FormattedErrorMessages.STORE_OP_FAILURE, "put", identifier, e); + throw new RuntimeException("Fail to import topic entity to store.", e); + } + } + + private EntityCombinedTopic internalLoadTopic(NameIdentifier ident) { + NameIdentifier catalogIdent = getCatalogIdentifier(ident); + Topic topic = + doWithCatalog( + catalogIdent, + c -> c.doWithTopicOps(t -> t.loadTopic(ident)), + NoSuchTopicException.class); + + StringIdentifier stringId = getStringIdFromProperties(topic.properties()); + if (stringId == null) { + return EntityCombinedTopic.of(topic) + .withHiddenPropertiesSet( + getHiddenPropertyNames( + catalogIdent, HasPropertyMetadata::topicPropertiesMetadata, topic.properties())) + .withImported(isEntityExist(ident, TOPIC)); + } + + TopicEntity topicEntity = + operateOnEntity( + ident, + identifier -> store.get(identifier, TOPIC, TopicEntity.class), + "GET", + getStringIdFromProperties(topic.properties()).id()); + + return EntityCombinedTopic.of(topic, topicEntity) + .withHiddenPropertiesSet( + getHiddenPropertyNames( + catalogIdent, HasPropertyMetadata::topicPropertiesMetadata, topic.properties())) + .withImported(topicEntity != null); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/config/ConfigBuilder.java b/core/src/main/java/com/datastrato/gravitino/config/ConfigBuilder.java index 90cbe5901af..dba14df70e1 100644 --- a/core/src/main/java/com/datastrato/gravitino/config/ConfigBuilder.java +++ b/core/src/main/java/com/datastrato/gravitino/config/ConfigBuilder.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.config; diff --git a/core/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java b/core/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java index d88e9682ece..6bf0671e127 100644 --- a/core/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java +++ b/core/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.config; @@ -34,4 +48,8 @@ private ConfigConstants() {} public static final String VERSION_0_5_0 = "0.5.0"; /** The version number for the 0.5.1 release. */ public static final String VERSION_0_5_1 = "0.5.1"; + /** The version number for the 0.5.2 release. */ + public static final String VERSION_0_5_2 = "0.5.2"; + /** The version number for the 0.6.0 release. */ + public static final String VERSION_0_6_0 = "0.6.0"; } diff --git a/core/src/main/java/com/datastrato/gravitino/config/ConfigEntry.java b/core/src/main/java/com/datastrato/gravitino/config/ConfigEntry.java index 3892f044c4b..7ee01ea8bb4 100644 --- a/core/src/main/java/com/datastrato/gravitino/config/ConfigEntry.java +++ b/core/src/main/java/com/datastrato/gravitino/config/ConfigEntry.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.config; @@ -140,7 +154,7 @@ public ConfigEntry checkValue(Function checkValueFunc, String err * Split the string to a list, then map each string element to its converted form. * * @param str The string form of the value list from the conf entry. - * @param converter The orignal ConfigEntry valueConverter. + * @param converter The original ConfigEntry valueConverter. * @return The list of converted type. */ public List strToSeq(String str, Function converter) { @@ -154,7 +168,7 @@ public List strToSeq(String str, Function converter) { * Reduce the values then join them as a string. * * @param seq The sequence of the value list from the conf entry. - * @param converter The orignal ConfigEntry stringConverter. + * @param converter The original ConfigEntry stringConverter. * @return The converted string. */ public String seqToStr(List seq, Function converter) { diff --git a/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalog.java b/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalog.java index 3f4fdb24144..bdf79827bf9 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalog.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalog.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalogPropertiesMetadata.java b/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalogPropertiesMetadata.java index bd211315388..2eada4e92f3 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalogPropertiesMetadata.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/BaseCatalogPropertiesMetadata.java @@ -1,12 +1,29 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; +import static com.datastrato.gravitino.Catalog.CLOUD_NAME; +import static com.datastrato.gravitino.Catalog.CLOUD_REGION_CODE; import static com.datastrato.gravitino.Catalog.PROPERTY_PACKAGE; +import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.annotation.Evolving; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; @@ -14,7 +31,6 @@ @Evolving public abstract class BaseCatalogPropertiesMetadata extends BasePropertiesMetadata { - protected static final Map> BASIC_CATALOG_PROPERTY_ENTRIES = Maps.uniqueIndex( ImmutableList.of( @@ -31,6 +47,21 @@ public abstract class BaseCatalogPropertiesMetadata extends BasePropertiesMetada false, null, false, - false)), + false), + PropertyEntry.enumPropertyEntry( + CLOUD_NAME, + "The cloud that the catalog is running on", + false /* required */, + true /* immutable */, + Catalog.CloudName.class, + null /* The default value does not work because if the user does not set it, this property will not be displayed */, + false /* hidden */, + false /* reserved */), + PropertyEntry.stringOptionalPropertyEntry( + CLOUD_REGION_CODE, + "The region code of the cloud that the catalog is running on", + false /* required */, + null /* The default value does not work because if the user does not set it, this property will not be displayed */, + false /* hidden */)), PropertyEntry::getName); } diff --git a/core/src/main/java/com/datastrato/gravitino/connector/BaseColumn.java b/core/src/main/java/com/datastrato/gravitino/connector/BaseColumn.java index ef335cde597..4f306e2bbed 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/BaseColumn.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/BaseColumn.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/BaseFileset.java b/core/src/main/java/com/datastrato/gravitino/connector/BaseFileset.java index 2b335a6c8c6..47f36e569ac 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/BaseFileset.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/BaseFileset.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/BasePropertiesMetadata.java b/core/src/main/java/com/datastrato/gravitino/connector/BasePropertiesMetadata.java index 2c535c3bf9a..c09299f3aad 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/BasePropertiesMetadata.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/BasePropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/BaseSchema.java b/core/src/main/java/com/datastrato/gravitino/connector/BaseSchema.java index 8b9e8acff8a..378ae8599ff 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/BaseSchema.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/BaseSchema.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/BaseTable.java b/core/src/main/java/com/datastrato/gravitino/connector/BaseTable.java index 6943294e964..133bcd62d01 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/BaseTable.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/BaseTable.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/BaseTopic.java b/core/src/main/java/com/datastrato/gravitino/connector/BaseTopic.java index f68fc233e45..e5a05480a2c 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/BaseTopic.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/BaseTopic.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/CatalogInfo.java b/core/src/main/java/com/datastrato/gravitino/connector/CatalogInfo.java index e6b820ec6e1..2529714b51b 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/CatalogInfo.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/CatalogInfo.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/CatalogOperations.java b/core/src/main/java/com/datastrato/gravitino/connector/CatalogOperations.java index 99ef0510cab..7e5224e17d0 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/CatalogOperations.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/CatalogOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/DataTypeConverter.java b/core/src/main/java/com/datastrato/gravitino/connector/DataTypeConverter.java new file mode 100644 index 00000000000..b55ed6d409f --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/connector/DataTypeConverter.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.connector; + +import com.datastrato.gravitino.rel.types.Type; + +/** + * The interface for converting data types between Gravitino and catalogs. In most cases, the ToType + * and FromType are the same. But in some cases, such as converting between Gravitino and JDBC + * types, the ToType is String and the FromType is JdbcTypeBean. + * + * @param The Gravitino type will be converted to. + * @param The type will be converted to Gravitino type. + */ +public interface DataTypeConverter { + /** + * Convert the Gravitino type to the catalog type. + * + * @param type The Gravitino type. + * @return The catalog type. + */ + ToType fromGravitino(Type type); + + /** + * Convert the catalog type to the Gravitino type. + * + * @param type The catalog type. + * @return The Gravitino type. + */ + Type toGravitino(FromType type); +} diff --git a/core/src/main/java/com/datastrato/gravitino/connector/HasPropertyMetadata.java b/core/src/main/java/com/datastrato/gravitino/connector/HasPropertyMetadata.java index 365de9bd336..50f13fffdc7 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/HasPropertyMetadata.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/HasPropertyMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/OperationsProxy.java b/core/src/main/java/com/datastrato/gravitino/connector/OperationsProxy.java index b25bc550d9e..2fa43069eec 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/OperationsProxy.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/OperationsProxy.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/PropertiesMetadata.java b/core/src/main/java/com/datastrato/gravitino/connector/PropertiesMetadata.java index 3343d52fab3..1a351a11a04 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/PropertiesMetadata.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/PropertiesMetadata.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/PropertyEntry.java b/core/src/main/java/com/datastrato/gravitino/connector/PropertyEntry.java index 450c54ab4ce..00d171c7b19 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/PropertyEntry.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/PropertyEntry.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/ProxyPlugin.java b/core/src/main/java/com/datastrato/gravitino/connector/ProxyPlugin.java index 40e1c0d794f..b4f0a7cd919 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/ProxyPlugin.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/ProxyPlugin.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/TableOperations.java b/core/src/main/java/com/datastrato/gravitino/connector/TableOperations.java index e215a836e09..1a65259a714 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/TableOperations.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/TableOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector; diff --git a/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java b/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java index e7ed63c6e92..cd313701fdc 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/capability/Capability.java @@ -1,11 +1,24 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector.capability; -import static com.datastrato.gravitino.Entity.SECURABLE_ENTITY_RESERVED_NAME; - +import com.datastrato.gravitino.MetadataObjects; import com.datastrato.gravitino.annotation.Evolving; import com.google.common.collect.ImmutableSet; import java.util.Set; @@ -82,7 +95,7 @@ default CapabilityResult managedStorage(Scope scope) { class DefaultCapability implements Capability { private static final Set RESERVED_WORDS = - ImmutableSet.of(SECURABLE_ENTITY_RESERVED_NAME); + ImmutableSet.of(MetadataObjects.METADATA_OBJECT_RESERVED_NAME); /** * Regular expression explanation: diff --git a/core/src/main/java/com/datastrato/gravitino/connector/capability/CapabilityResult.java b/core/src/main/java/com/datastrato/gravitino/connector/capability/CapabilityResult.java index 629c31caf4e..84e92f08aff 100644 --- a/core/src/main/java/com/datastrato/gravitino/connector/capability/CapabilityResult.java +++ b/core/src/main/java/com/datastrato/gravitino/connector/capability/CapabilityResult.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.connector.capability; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/AsyncQueueListener.java b/core/src/main/java/com/datastrato/gravitino/listener/AsyncQueueListener.java index d9beb624351..635eb11db4b 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/AsyncQueueListener.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/AsyncQueueListener.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/CatalogEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/listener/CatalogEventDispatcher.java index febad0a8c7e..89f45e1ba9d 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/CatalogEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/CatalogEventDispatcher.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/EventBus.java b/core/src/main/java/com/datastrato/gravitino/listener/EventBus.java index 0c186f4d003..f7015b65a84 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/EventBus.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/EventBus.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/EventListenerConfig.java b/core/src/main/java/com/datastrato/gravitino/listener/EventListenerConfig.java index f8032031758..85dce77c2ef 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/EventListenerConfig.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/EventListenerConfig.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/EventListenerManager.java b/core/src/main/java/com/datastrato/gravitino/listener/EventListenerManager.java index c26e9abcdbd..87ac099fc31 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/EventListenerManager.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/EventListenerManager.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/EventListenerPluginWrapper.java b/core/src/main/java/com/datastrato/gravitino/listener/EventListenerPluginWrapper.java index 2dda84a84b1..902a39491a7 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/EventListenerPluginWrapper.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/EventListenerPluginWrapper.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/FilesetEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/listener/FilesetEventDispatcher.java index 5ed8b6462f8..7c84fb552f6 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/FilesetEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/FilesetEventDispatcher.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/MetalakeEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/listener/MetalakeEventDispatcher.java index 8c3d79d7da2..555f2fbc8a2 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/MetalakeEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/MetalakeEventDispatcher.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/SchemaEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/listener/SchemaEventDispatcher.java index 50c6df4ea18..a1bfbc98ccf 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/SchemaEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/SchemaEventDispatcher.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/TableEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/listener/TableEventDispatcher.java index deacbba2094..30f2fe0a364 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/TableEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/TableEventDispatcher.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/TopicEventDispatcher.java b/core/src/main/java/com/datastrato/gravitino/listener/TopicEventDispatcher.java index dd628a534d2..4d93ac310c9 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/TopicEventDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/TopicEventDispatcher.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/EventListenerPlugin.java b/core/src/main/java/com/datastrato/gravitino/listener/api/EventListenerPlugin.java index 24bebbac30f..ffb480debfa 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/EventListenerPlugin.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/EventListenerPlugin.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogEvent.java index 8d712f75cf3..a5a5fa9c120 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogFailureEvent.java index a47781cdf10..404a8ad17a1 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterCatalogFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetEvent.java index 57fed2e8db6..db193d47755 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetFailureEvent.java index 515f8cdce92..a9970ed4906 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterFilesetFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeEvent.java index ba1baaafe86..d0b8e0fc6e2 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeFailureEvent.java index dd58aad26ab..c16b13f97dc 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterMetalakeFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaEvent.java index 5e180ad3c98..2353148a49f 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaFailureEvent.java index d55d5c89d88..07a354f570a 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterSchemaFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableEvent.java index a18359c70f3..a3abb92203d 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableFailureEvent.java index 0c8b1edbfbe..432e7e5badc 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTableFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicEvent.java index c6a0b6961b1..bb8a340fe6e 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicFailureEvent.java index dc207450260..5e61cb6f7c2 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/AlterTopicFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogEvent.java index 03223c90ff2..e6926d8b092 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogFailureEvent.java index 80fbbffb167..6e48a6a83e4 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CatalogFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogEvent.java index 8bc04f1895e..01f3d7833be 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogFailureEvent.java index 2194ce0b9bc..c8bbe95ef77 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateCatalogFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetEvent.java index d5e091dd725..4db862b35b6 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetFailureEvent.java index 6e2a49c0a01..4a1b5eb6a19 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateFilesetFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeEvent.java index b3c9c4a1f50..4c6673a8a02 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeFailureEvent.java index 64fa352cd13..39bf4bb6680 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateMetalakeFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaEvent.java index 4c5f08e475b..64e052f9278 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaFailureEvent.java index 5215d1b623d..3af7f02cf87 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateSchemaFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableEvent.java index fe89dcc8f63..f1d5a721bcd 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableFailureEvent.java index dc60ad44c88..bedbc7ce263 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTableFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicEvent.java index f3418faf0f8..991b08faf79 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicFailureEvent.java index 3ec9425a941..519ee702038 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/CreateTopicFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogEvent.java index bf2f01effbf..df466c3cea4 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogFailureEvent.java index 48319ed8e45..07d85de17ca 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropCatalogFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetEvent.java index dd65eac931c..64bd6bd1878 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetFailureEvent.java index dcfb8b733ca..a678ea12138 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropFilesetFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeEvent.java index 944bed7a9de..7137e782a12 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeFailureEvent.java index 4a70b33a14c..9571c6a2580 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropMetalakeFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaEvent.java index 6c1dbbda917..0d5742f5919 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaFailureEvent.java index 5fa43362e8d..0c9a8118095 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropSchemaFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableEvent.java index 6e2ba0a6dd5..4bf28ab9929 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableFailureEvent.java index 87eef3b9e22..62cb3230afe 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTableFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicEvent.java index 70051e6ae28..91f9b2019a0 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicFailureEvent.java index 5668783a660..dfe05ded93b 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/DropTopicFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java index b0abffeb682..2ca28a41da4 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/Event.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/FailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FailureEvent.java index 471c9baad9f..3528157d9ba 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/FailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetEvent.java index 2e9e0722c1a..5dc6f11a7b0 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetFailureEvent.java index 010d25c36c3..8f560722349 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/FilesetFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogEvent.java index 2e1421ac09e..5f29e91f354 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogFailureEvent.java index 203310fde74..2442068aaf3 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListCatalogFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetEvent.java index 5d188631466..43fc71b2a6e 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetFailureEvent.java index 745945b3847..69519f6a5f4 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListFilesetFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeEvent.java index 1d72b6477f4..56414c2809c 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeFailureEvent.java index bd065e909c7..b75500cf08a 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListMetalakeFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaEvent.java index 08aa0b38a33..4abf59bc496 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaFailureEvent.java index b1c0a037ae4..c07c51f98db 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListSchemaFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableEvent.java index 7a28ebe08e1..4d01546d63b 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableFailureEvent.java index 169b6ea80d1..6de7383641a 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTableFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicEvent.java index 16297fe385b..1f3f0792203 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicFailureEvent.java index a7c686f6256..30cd97df0d7 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/ListTopicFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogEvent.java index f07d2cda6d3..6889a72f186 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogFailureEvent.java index 3972fb0526c..6c03cf41daf 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadCatalogFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetEvent.java index 8d7d238ff9d..270b6c2334d 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetFailureEvent.java index 724bd384b88..fde81a9f86e 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadFilesetFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeEvent.java index 10827eb081e..a09ff1f8f9c 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeFailureEvent.java index 51cda87dd08..88eedce2737 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadMetalakeFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaEvent.java index c9418923020..5331b12e376 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaFailureEvent.java index c552568233a..ab76b1f7581 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadSchemaFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableEvent.java index f3bfb4236dd..19d7e5ae6ca 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableFailureEvent.java index b01b8493207..84490677276 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTableFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicEvent.java index ce516ffa9c8..6cbe8a62d73 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicFailureEvent.java index ee3f5b6401e..b022da23bee 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/LoadTopicFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeEvent.java index 7820003c023..f4a8a5524f7 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeFailureEvent.java index 01637c6aeea..5297d4b2854 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/MetalakeFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableEvent.java index fd3048ee460..bb55c26e970 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableFailureEvent.java index 42fddfc0f1c..1431625403f 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/PurgeTableFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaEvent.java index 60b667afb38..7f5c03fb221 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaFailureEvent.java index 3c0cd1e4af8..e16f5e880dc 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/SchemaFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableEvent.java index 19e10d5f229..afdf0d9984a 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableFailureEvent.java index e87d1281fa1..f864cd8ca72 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TableFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicEvent.java index f6fadae14aa..661b245072b 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicFailureEvent.java b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicFailureEvent.java index a05b45862d8..6f8483f8b53 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicFailureEvent.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/event/TopicFailureEvent.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.event; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/CatalogInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/CatalogInfo.java index 0e1f64f9272..70972221ecd 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/info/CatalogInfo.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/CatalogInfo.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.info; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/FilesetInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/FilesetInfo.java index 93b0f1e5aba..45bb3f249f3 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/info/FilesetInfo.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/FilesetInfo.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.info; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/MetalakeInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/MetalakeInfo.java index d17566b1929..32e745e6130 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/info/MetalakeInfo.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/MetalakeInfo.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.info; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/SchemaInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/SchemaInfo.java index 7d4cd8ccbdb..96e89f18820 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/info/SchemaInfo.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/SchemaInfo.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.info; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/TableInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/TableInfo.java index a877c00af24..89e9bb32f77 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/info/TableInfo.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/TableInfo.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.info; diff --git a/core/src/main/java/com/datastrato/gravitino/listener/api/info/TopicInfo.java b/core/src/main/java/com/datastrato/gravitino/listener/api/info/TopicInfo.java index 2c5f8bd05dc..fd3b0092d09 100644 --- a/core/src/main/java/com/datastrato/gravitino/listener/api/info/TopicInfo.java +++ b/core/src/main/java/com/datastrato/gravitino/listener/api/info/TopicInfo.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.listener.api.info; diff --git a/core/src/main/java/com/datastrato/gravitino/lock/LockManager.java b/core/src/main/java/com/datastrato/gravitino/lock/LockManager.java index e03d75e1acb..b1dbb27fed2 100644 --- a/core/src/main/java/com/datastrato/gravitino/lock/LockManager.java +++ b/core/src/main/java/com/datastrato/gravitino/lock/LockManager.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.lock; diff --git a/core/src/main/java/com/datastrato/gravitino/lock/LockType.java b/core/src/main/java/com/datastrato/gravitino/lock/LockType.java index 7a68cb54ca4..d619e94cb86 100644 --- a/core/src/main/java/com/datastrato/gravitino/lock/LockType.java +++ b/core/src/main/java/com/datastrato/gravitino/lock/LockType.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.lock; diff --git a/core/src/main/java/com/datastrato/gravitino/lock/TreeLock.java b/core/src/main/java/com/datastrato/gravitino/lock/TreeLock.java index ec8076ff752..76d9ab02887 100644 --- a/core/src/main/java/com/datastrato/gravitino/lock/TreeLock.java +++ b/core/src/main/java/com/datastrato/gravitino/lock/TreeLock.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.lock; diff --git a/core/src/main/java/com/datastrato/gravitino/lock/TreeLockNode.java b/core/src/main/java/com/datastrato/gravitino/lock/TreeLockNode.java index 220050db62c..a4953c54104 100644 --- a/core/src/main/java/com/datastrato/gravitino/lock/TreeLockNode.java +++ b/core/src/main/java/com/datastrato/gravitino/lock/TreeLockNode.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.lock; diff --git a/core/src/main/java/com/datastrato/gravitino/lock/TreeLockUtils.java b/core/src/main/java/com/datastrato/gravitino/lock/TreeLockUtils.java index 2826555defc..31232e1e6eb 100644 --- a/core/src/main/java/com/datastrato/gravitino/lock/TreeLockUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/lock/TreeLockUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.lock; diff --git a/core/src/main/java/com/datastrato/gravitino/meta/AuditInfo.java b/core/src/main/java/com/datastrato/gravitino/meta/AuditInfo.java index 48661018059..43d0c331347 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/AuditInfo.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/AuditInfo.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.meta; diff --git a/core/src/main/java/com/datastrato/gravitino/meta/BaseMetalake.java b/core/src/main/java/com/datastrato/gravitino/meta/BaseMetalake.java index f4c22470057..ca5981b83e3 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/BaseMetalake.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/BaseMetalake.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.meta; diff --git a/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java index f745e6fddc7..2a327808c55 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.meta; diff --git a/core/src/main/java/com/datastrato/gravitino/meta/FilesetEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/FilesetEntity.java index 0f8726a313b..e34e4c780d3 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/FilesetEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/FilesetEntity.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.meta; diff --git a/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java index f9c3b9cd881..72b3726d2a1 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.meta; diff --git a/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java index aaa8c15b36b..e1101ce9bf9 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.meta; @@ -11,7 +25,7 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.authorization.Role; import com.datastrato.gravitino.authorization.SecurableObject; -import com.google.common.collect.Lists; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import java.util.Collections; import java.util.List; @@ -33,15 +47,14 @@ public class RoleEntity implements Role, Entity, Auditable, HasIdentifier { Field.required("audit_info", AuditInfo.class, "The audit details of the role entity."); public static final Field SECURABLE_OBJECT = - Field.required( - "securable_object", SecurableObject.class, "The securable object of the role entity."); + Field.required("securable_objects", List.class, "The securable objects of the role entity."); private Long id; private String name; private Map properties; private AuditInfo auditInfo; private Namespace namespace; - private SecurableObject securableObject; + private List securableObjects; /** * The name of the role. @@ -87,7 +100,7 @@ public List securableObjects() { // So one type of them can't be the securable object at least if there are the two same // identifier // entities . - return Lists.newArrayList(securableObject); + return securableObjects; } /** @@ -102,7 +115,7 @@ public Map fields() { fields.put(NAME, name); fields.put(AUDIT_INFO, auditInfo); fields.put(PROPERTIES, properties); - fields.put(SECURABLE_OBJECT, securableObject); + fields.put(SECURABLE_OBJECT, securableObjects); return Collections.unmodifiableMap(fields); } @@ -138,12 +151,12 @@ public boolean equals(Object o) { && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) && Objects.equals(properties, that.properties) - && Objects.equals(securableObject, that.securableObject); + && Objects.equals(securableObjects, that.securableObjects); } @Override public int hashCode() { - return Objects.hash(id, name, properties, auditInfo, securableObject); + return Objects.hash(id, name, properties, auditInfo, securableObjects); } /** @@ -212,13 +225,13 @@ public Builder withAuditInfo(AuditInfo auditInfo) { } /** - * Sets the securable object of the role entity. + * Sets the securable objects of the role entity. * - * @param securableObject The securable object of the role entity. + * @param securableObjects The securable objects of the role entity. * @return The builder instance. */ - public Builder withSecurableObject(SecurableObject securableObject) { - roleEntity.securableObject = securableObject; + public Builder withSecurableObjects(List securableObjects) { + roleEntity.securableObjects = ImmutableList.copyOf(securableObjects); return this; } diff --git a/core/src/main/java/com/datastrato/gravitino/meta/SchemaEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/SchemaEntity.java index 46e747992ea..5c69b13dcd8 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/SchemaEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/SchemaEntity.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.meta; diff --git a/core/src/main/java/com/datastrato/gravitino/meta/SchemaVersion.java b/core/src/main/java/com/datastrato/gravitino/meta/SchemaVersion.java index 5faa86db4cf..ff44afac16e 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/SchemaVersion.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/SchemaVersion.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.meta; diff --git a/core/src/main/java/com/datastrato/gravitino/meta/TableEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/TableEntity.java index 37d1ee5a984..24cda485cbf 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/TableEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/TableEntity.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.meta; @@ -105,7 +119,6 @@ public boolean equals(Object o) { return false; } - // Ignore field namespace TableEntity baseTable = (TableEntity) o; return Objects.equal(id, baseTable.id) && Objects.equal(name, baseTable.name) diff --git a/core/src/main/java/com/datastrato/gravitino/meta/TagEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/TagEntity.java new file mode 100644 index 00000000000..b4acaf71f3d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/meta/TagEntity.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.datastrato.gravitino.meta; + +import com.datastrato.gravitino.Audit; +import com.datastrato.gravitino.Auditable; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.Field; +import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.tag.Tag; +import com.google.common.collect.Maps; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +public class TagEntity implements Tag, Entity, Auditable, HasIdentifier { + + public static final Field ID = + Field.required("id", Long.class, "The unique id of the tag entity."); + + public static final Field NAME = + Field.required("name", String.class, "The name of the tag entity."); + + public static final Field COMMENT = + Field.optional("comment", String.class, "The comment of the tag entity."); + + public static final Field PROPERTIES = + Field.optional("properties", Map.class, "The properties of the tag entity."); + + public static final Field ASSOCIATED_OBJECTS = + Field.optional( + "objects", MetadataObject[].class, "The associated objects of the tag entity."); + + public static final Field AUDIT_INFO = + Field.required("audit_info", Audit.class, "The audit details of the tag entity."); + + private Long id; + private String name; + private Namespace namespace; + private String comment; + private Map properties; + private MetadataObject[] objects = null; + private Audit auditInfo; + + private TagEntity() {} + + @Override + public Map fields() { + Map fields = Maps.newHashMap(); + fields.put(ID, id); + fields.put(NAME, name); + fields.put(COMMENT, comment); + fields.put(PROPERTIES, properties); + fields.put(AUDIT_INFO, auditInfo); + fields.put(ASSOCIATED_OBJECTS, objects); + + return Collections.unmodifiableMap(fields); + } + + @Override + public EntityType type() { + return EntityType.TAG; + } + + @Override + public Long id() { + return id; + } + + @Override + public String name() { + return name; + } + + @Override + public Namespace namespace() { + return namespace; + } + + @Override + public String comment() { + return comment; + } + + @Override + public Map properties() { + return properties; + } + + @Override + public Optional inherited() { + return Optional.empty(); + } + + public MetadataObject[] objects() { + return objects; + } + + @Override + public Audit auditInfo() { + return auditInfo; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TagEntity)) { + return false; + } + + TagEntity that = (TagEntity) o; + return Objects.equals(id, that.id) + && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) + && Objects.equals(comment, that.comment) + && Objects.equals(properties, that.properties) + && Objects.equals(auditInfo, that.auditInfo); + } + + @Override + public int hashCode() { + return Objects.hash(id, name, namespace, comment, properties, auditInfo); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private final TagEntity tagEntity; + + private Builder() { + this.tagEntity = new TagEntity(); + } + + public Builder withId(Long id) { + tagEntity.id = id; + return this; + } + + public Builder withName(String name) { + tagEntity.name = name; + return this; + } + + public Builder withNamespace(Namespace namespace) { + tagEntity.namespace = namespace; + return this; + } + + public Builder withComment(String comment) { + tagEntity.comment = comment; + return this; + } + + public Builder withProperties(Map properties) { + tagEntity.properties = properties; + return this; + } + + public Builder withMetadataObjects(MetadataObject[] objects) { + tagEntity.objects = objects; + return this; + } + + public Builder withAuditInfo(Audit auditInfo) { + tagEntity.auditInfo = auditInfo; + return this; + } + + public TagEntity build() { + tagEntity.validate(); + return tagEntity; + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/meta/TopicEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/TopicEntity.java index 898e9b35bba..02780d9acd7 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/TopicEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/TopicEntity.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.meta; diff --git a/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java index 954ec9924e3..cff4694b863 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.meta; diff --git a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeDispatcher.java index 645c7220296..f565df361a7 100644 --- a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeDispatcher.java @@ -1,6 +1,19 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.metalake; diff --git a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java index 5542ef0d0ed..f707a50cacd 100644 --- a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java +++ b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.metalake; diff --git a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeNormalizeDispatcher.java b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeNormalizeDispatcher.java index fe11a7ec406..82c828d78e1 100644 --- a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeNormalizeDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeNormalizeDispatcher.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.metalake; diff --git a/core/src/main/java/com/datastrato/gravitino/metalake/SupportsMetalakes.java b/core/src/main/java/com/datastrato/gravitino/metalake/SupportsMetalakes.java index 43b0b1eb11c..f8d038b38f0 100644 --- a/core/src/main/java/com/datastrato/gravitino/metalake/SupportsMetalakes.java +++ b/core/src/main/java/com/datastrato/gravitino/metalake/SupportsMetalakes.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.metalake; diff --git a/core/src/main/java/com/datastrato/gravitino/metrics/MetricNames.java b/core/src/main/java/com/datastrato/gravitino/metrics/MetricNames.java index 21502c09e88..dfe8c3f8507 100644 --- a/core/src/main/java/com/datastrato/gravitino/metrics/MetricNames.java +++ b/core/src/main/java/com/datastrato/gravitino/metrics/MetricNames.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.metrics; diff --git a/core/src/main/java/com/datastrato/gravitino/metrics/MetricsSystem.java b/core/src/main/java/com/datastrato/gravitino/metrics/MetricsSystem.java index 7b699716b95..955cd348b20 100644 --- a/core/src/main/java/com/datastrato/gravitino/metrics/MetricsSystem.java +++ b/core/src/main/java/com/datastrato/gravitino/metrics/MetricsSystem.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.metrics; diff --git a/core/src/main/java/com/datastrato/gravitino/metrics/source/JVMMetricsSource.java b/core/src/main/java/com/datastrato/gravitino/metrics/source/JVMMetricsSource.java index 5386f9754f9..7e5e81f35ce 100644 --- a/core/src/main/java/com/datastrato/gravitino/metrics/source/JVMMetricsSource.java +++ b/core/src/main/java/com/datastrato/gravitino/metrics/source/JVMMetricsSource.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.metrics.source; diff --git a/core/src/main/java/com/datastrato/gravitino/metrics/source/MetricsSource.java b/core/src/main/java/com/datastrato/gravitino/metrics/source/MetricsSource.java index 4ff58eda0d4..627f309f7fa 100644 --- a/core/src/main/java/com/datastrato/gravitino/metrics/source/MetricsSource.java +++ b/core/src/main/java/com/datastrato/gravitino/metrics/source/MetricsSource.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.metrics.source; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/AuditInfoSerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/AuditInfoSerDe.java index 7891af5ccc5..723252c26cd 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/AuditInfoSerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/AuditInfoSerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/BaseMetalakeSerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/BaseMetalakeSerDe.java index 0f9f09153dd..0379bbcbc66 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/BaseMetalakeSerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/BaseMetalakeSerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/CatalogEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/CatalogEntitySerDe.java index 382761da4d9..77381fdfc92 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/CatalogEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/CatalogEntitySerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/FilesetEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/FilesetEntitySerDe.java index 24680452739..0f3464415a1 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/FilesetEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/FilesetEntitySerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java index fb5465802d4..6cd79b67bee 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java index 894607d7dbf..6188bfc948b 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDe.java index 33e44b60fa4..e8224da1013 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDeException.java b/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDeException.java index 10d8e11bbc1..60f240f1371 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDeException.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDeException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/ProtoUtils.java b/core/src/main/java/com/datastrato/gravitino/proto/ProtoUtils.java index c7a545dbdbb..6f2aca71567 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/ProtoUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/ProtoUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java index e0add5b3f03..851cd0f2955 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; @@ -28,17 +42,25 @@ public Role serialize(RoleEntity roleEntity) { Role.newBuilder() .setId(roleEntity.id()) .setName(roleEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(roleEntity.auditInfo())) - .addAllPrivileges( - roleEntity.securableObjects().get(0).privileges().stream() - .map(privilege -> privilege.name().toString()) - .collect(Collectors.toList())) - .addAllPrivilegeConditions( - roleEntity.securableObjects().get(0).privileges().stream() - .map(privilege -> privilege.condition().toString()) - .collect(Collectors.toList())) - .setSecurableObjectFullName(roleEntity.securableObjects().get(0).fullName()) - .setSecurableObjectType(roleEntity.securableObjects().get(0).type().name()); + .setAuditInfo(new AuditInfoSerDe().serialize(roleEntity.auditInfo())); + + for (SecurableObject securableObject : roleEntity.securableObjects()) { + builder.addSecurableObjects( + com.datastrato.gravitino.proto.SecurableObject.newBuilder() + .setFullName(securableObject.fullName()) + .setType(securableObject.type().name()) + .addAllPrivilegeConditions( + securableObject.privileges().stream() + .map(Privilege::condition) + .map(Privilege.Condition::name) + .collect(Collectors.toList())) + .addAllPrivilegeNames( + securableObject.privileges().stream() + .map(Privilege::name) + .map(Privilege.Name::name) + .collect(Collectors.toList())) + .build()); + } if (roleEntity.properties() != null && !roleEntity.properties().isEmpty()) { builder.putAllProperties(roleEntity.properties()); @@ -55,31 +77,32 @@ public Role serialize(RoleEntity roleEntity) { */ @Override public RoleEntity deserialize(Role role, Namespace namespace) { - List privileges = Lists.newArrayList(); - - if (!role.getPrivilegesList().isEmpty()) { + List securableObjects = Lists.newArrayList(); - for (int index = 0; index < role.getPrivilegeConditionsCount(); index++) { - if (Privilege.Condition.ALLOW.name().equals(role.getPrivilegeConditions(index))) { - privileges.add(Privileges.allow(role.getPrivileges(index))); + for (int index = 0; index < role.getSecurableObjectsCount(); index++) { + List privileges = Lists.newArrayList(); + com.datastrato.gravitino.proto.SecurableObject object = role.getSecurableObjects(index); + for (int privIndex = 0; privIndex < object.getPrivilegeConditionsCount(); privIndex++) { + if (Privilege.Condition.ALLOW.name().equals(object.getPrivilegeConditions(privIndex))) { + privileges.add(Privileges.allow(object.getPrivilegeNames(privIndex))); } else { - privileges.add(Privileges.deny(role.getPrivileges(index))); + privileges.add(Privileges.deny(object.getPrivilegeNames(privIndex))); } } - } - SecurableObject securableObject = - SecurableObjects.parse( - role.getSecurableObjectFullName(), - SecurableObject.Type.valueOf(role.getSecurableObjectType()), - privileges); + SecurableObject securableObject = + SecurableObjects.parse( + object.getFullName(), SecurableObject.Type.valueOf(object.getType()), privileges); + + securableObjects.add(securableObject); + } RoleEntity.Builder builder = RoleEntity.builder() .withId(role.getId()) .withName(role.getName()) .withNamespace(namespace) - .withSecurableObject(securableObject) + .withSecurableObjects(securableObjects) .withAuditInfo(new AuditInfoSerDe().deserialize(role.getAuditInfo(), namespace)); if (!role.getPropertiesMap().isEmpty()) { diff --git a/core/src/main/java/com/datastrato/gravitino/proto/SchemaEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/SchemaEntitySerDe.java index 1df134bbd4e..b4119e39cbb 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/SchemaEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/SchemaEntitySerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/TableEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/TableEntitySerDe.java index b2d79c4f7c9..269ed473f06 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/TableEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/TableEntitySerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/TopicEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/TopicEntitySerDe.java index 1fb6ce92275..661a791e3eb 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/TopicEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/TopicEntitySerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java index c839b574e3a..ba4cc0d786f 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.proto; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/BiPredicate.java b/core/src/main/java/com/datastrato/gravitino/storage/BiPredicate.java index 710687ac980..2b5af250bd1 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/BiPredicate.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/BiPredicate.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/EntityKeyEncoder.java b/core/src/main/java/com/datastrato/gravitino/storage/EntityKeyEncoder.java index fcbea84c2c7..cf3167f09e8 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/EntityKeyEncoder.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/EntityKeyEncoder.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/FunctionUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/FunctionUtils.java index 363b1ce8e9e..90bbc5ccdcb 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/FunctionUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/FunctionUtils.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/IdGenerator.java b/core/src/main/java/com/datastrato/gravitino/storage/IdGenerator.java index efb376c9d9b..2f4653b5897 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/IdGenerator.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/IdGenerator.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/NameMappingService.java b/core/src/main/java/com/datastrato/gravitino/storage/NameMappingService.java index 83136bb22c4..f368352cd83 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/NameMappingService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/NameMappingService.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/RandomIdGenerator.java b/core/src/main/java/com/datastrato/gravitino/storage/RandomIdGenerator.java index 26def2b8ec4..7f457887df0 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/RandomIdGenerator.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/RandomIdGenerator.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/StorageLayoutException.java b/core/src/main/java/com/datastrato/gravitino/storage/StorageLayoutException.java index 5e24896c3c9..719cc1abcc3 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/StorageLayoutException.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/StorageLayoutException.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/StorageLayoutVersion.java b/core/src/main/java/com/datastrato/gravitino/storage/StorageLayoutVersion.java index f4696714884..8d62ae0d4a1 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/StorageLayoutVersion.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/StorageLayoutVersion.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/TransactionIdGenerator.java b/core/src/main/java/com/datastrato/gravitino/storage/TransactionIdGenerator.java index 70dd4ebefac..d6a8f174271 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/TransactionIdGenerator.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/TransactionIdGenerator.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityEncoderUtil.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityEncoderUtil.java index 28bd3614070..916217bf52d 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityEncoderUtil.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityEncoderUtil.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.kv; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityKeyEncoder.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityKeyEncoder.java index 49fea351a69..94f0a073119 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityKeyEncoder.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityKeyEncoder.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.kv; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvBackend.java index f2d2d4be55c..b1d39b8f15a 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvBackend.java @@ -1,12 +1,26 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.kv; import com.datastrato.gravitino.Config; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; +import com.datastrato.gravitino.EntityAlreadyExistsException; import java.io.Closeable; import java.io.IOException; import java.util.List; @@ -29,9 +43,10 @@ public interface KvBackend extends Closeable { * @param value The value of the pair. * @param overwrite If true, overwrites the existing value. * @throws IOException If an I/O exception occurs during the operation. - * @throws AlreadyExistsException If the key already exists and overwrite is false. + * @throws EntityAlreadyExistsException If the key already exists and overwrite is false. */ - void put(byte[] key, byte[] value, boolean overwrite) throws IOException, AlreadyExistsException; + void put(byte[] key, byte[] value, boolean overwrite) + throws IOException, EntityAlreadyExistsException; /** * Retrieves the value associated with a given key. diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java index 917558888ab..66cd273ab63 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.kv; @@ -24,7 +38,6 @@ import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.exceptions.NonEmptyEntityException; import com.datastrato.gravitino.storage.EntityKeyEncoder; @@ -168,7 +181,7 @@ public void put(E e, boolean overwritten) @Override public E update( NameIdentifier ident, Class type, EntityType entityType, Function updater) - throws IOException, NoSuchEntityException, AlreadyExistsException { + throws IOException, NoSuchEntityException, EntityAlreadyExistsException { return executeInTransaction( () -> { byte[] key = entityKeyEncoder.encode(ident, entityType); @@ -188,7 +201,7 @@ public E update( // Check whether the new entities already existed boolean newEntityExist = exists(updatedE.nameIdentifier(), entityType); if (newEntityExist) { - throw new AlreadyExistsException( + throw new EntityAlreadyExistsException( "Entity %s already exist, please check again", updatedE.nameIdentifier()); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java index 309323f490c..48d4db72ecf 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.kv; @@ -76,11 +90,11 @@ public KvGarbageCollector( } public void start() { - long dateTimeLineMinute = config.get(STORE_DELETE_AFTER_TIME) / 1000 / 60; + long dateTimelineMinute = config.get(STORE_DELETE_AFTER_TIME) / 1000 / 60; - // We will collect garbage every 10 minutes at least. If the dateTimeLineMinute is larger than - // 100 minutes, we would collect garbage every dateTimeLineMinute/10 minutes. - this.frequencyInMinutes = Math.max(dateTimeLineMinute / 10, 10); + // We will collect garbage every 10 minutes at least. If the dateTimelineMinute is larger than + // 100 minutes, we would collect garbage every dateTimelineMinute/10 minutes. + this.frequencyInMinutes = Math.max(dateTimelineMinute / 10, 10); garbageCollectorPool.scheduleAtFixedRate( this::collectAndClean, 5, frequencyInMinutes, TimeUnit.MINUTES); } @@ -141,11 +155,11 @@ private void collectAndRemoveUncommittedData() throws IOException { } private void collectAndRemoveOldVersionData() throws IOException { - long deleteTimeLine = System.currentTimeMillis() - config.get(STORE_DELETE_AFTER_TIME); + long deleteTimeline = System.currentTimeMillis() - config.get(STORE_DELETE_AFTER_TIME); // Why should we leave shift 18 bits? please refer to TransactionIdGeneratorImpl#nextId - // We can delete the data which is older than deleteTimeLine.(old data with transaction id that + // We can delete the data which is older than deleteTimeline.(old data with transaction id that // is smaller than transactionIdToDelete) - long transactionIdToDelete = deleteTimeLine << 18; + long transactionIdToDelete = deleteTimeline << 18; LOG.info("Start to remove data which is older than {}", transactionIdToDelete); byte[] startKey = TransactionalKvBackendImpl.generateCommitKey(transactionIdToDelete); commitIdHasBeenCollected = kvBackend.get(LAST_COLLECT_COMMIT_ID_KEY); @@ -159,7 +173,7 @@ private void collectAndRemoveOldVersionData() throws IOException { lastGCId, lastGCId == 1 ? lastGCId : DateFormatUtils.format(lastGCId >> 18, TIME_STAMP_FORMAT), transactionIdToDelete, - DateFormatUtils.format(deleteTimeLine, TIME_STAMP_FORMAT)); + DateFormatUtils.format(deleteTimeline, TIME_STAMP_FORMAT)); // Get all commit marks // TODO(yuqi), Use multi-thread to scan the data in case of the data is too large. diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvNameMappingService.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvNameMappingService.java index f0050092b56..01bf818a91e 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvNameMappingService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvNameMappingService.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.kv; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvRange.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvRange.java index f2916fb2b4e..2487c2218c1 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvRange.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvRange.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.kv; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/RocksDBKvBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/RocksDBKvBackend.java index 97dcd29f5ed..c7079d7b7d6 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/RocksDBKvBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/RocksDBKvBackend.java @@ -1,13 +1,27 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.kv; import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Configs; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; +import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.utils.ByteUtils; import com.datastrato.gravitino.utils.Bytes; import com.google.common.annotations.VisibleForTesting; @@ -67,7 +81,7 @@ private RocksDB initRocksDB(Config config) throws RocksDBException { @VisibleForTesting String getStoragePath(Config config) { - String dbPath = config.get(Configs.ENTRY_KV_ROCKSDB_BACKEND_PATH); + String dbPath = config.get(Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH); if (StringUtils.isBlank(dbPath)) { return Configs.DEFAULT_KV_ROCKSDB_BACKEND_PATH; } @@ -95,7 +109,7 @@ public void initialize(Config config) throws IOException { public void put(byte[] key, byte[] value, boolean overwrite) throws IOException { try { handlePut(key, value, overwrite); - } catch (AlreadyExistsException e) { + } catch (EntityAlreadyExistsException e) { throw e; } catch (Exception e) { throw new IOException(e); @@ -110,7 +124,7 @@ void handlePut(byte[] key, byte[] value, boolean overwrite) throws RocksDBExcept } byte[] existKey = db.get(key); if (existKey != null) { - throw new AlreadyExistsException( + throw new EntityAlreadyExistsException( "Key %s already exists in the database, please use overwrite option to overwrite it", ByteUtils.formatByteArray(key)); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/TransactionIdGeneratorImpl.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/TransactionIdGeneratorImpl.java index 90e8844089f..208ce2ddded 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/TransactionIdGeneratorImpl.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/TransactionIdGeneratorImpl.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.kv; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/TransactionalKvBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/TransactionalKvBackend.java index c9b3724a30c..04f01d45d86 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/TransactionalKvBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/TransactionalKvBackend.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.kv; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/TransactionalKvBackendImpl.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/TransactionalKvBackendImpl.java index 1ab4b4062fa..96175cf150f 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/TransactionalKvBackendImpl.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/TransactionalKvBackendImpl.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.kv; @@ -142,7 +156,7 @@ public void put(byte[] key, byte[] value, boolean overwrite) throws IOException, EntityAlreadyExistsException { byte[] oldValue = get(key); if (oldValue != null && !overwrite) { - throw new EntityAlreadyExistsException("Key already exists: " + Bytes.wrap(key)); + throw new EntityAlreadyExistsException("Key already exists: %s", Bytes.wrap(key)); } putPairs .get() diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/ValueStatusEnum.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/ValueStatusEnum.java index badc222b46d..624f622c3d9 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/ValueStatusEnum.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/ValueStatusEnum.java @@ -1,6 +1,20 @@ /* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.kv; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index 8c261d4edb2..ea574339a03 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational; @@ -15,7 +29,6 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.UnsupportedEntityTypeException; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.CatalogEntity; @@ -27,6 +40,7 @@ import com.datastrato.gravitino.meta.TopicEntity; import com.datastrato.gravitino.meta.UserEntity; import com.datastrato.gravitino.storage.relational.converters.SQLExceptionConverterFactory; +import com.datastrato.gravitino.storage.relational.database.H2Database; import com.datastrato.gravitino.storage.relational.service.CatalogMetaService; import com.datastrato.gravitino.storage.relational.service.FilesetMetaService; import com.datastrato.gravitino.storage.relational.service.GroupMetaService; @@ -37,8 +51,10 @@ import com.datastrato.gravitino.storage.relational.service.TopicMetaService; import com.datastrato.gravitino.storage.relational.service.UserMetaService; import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.function.Function; /** @@ -49,16 +65,24 @@ */ public class JDBCBackend implements RelationalBackend { + private static final Map EMBEDDED_JDBC_DATABASE_MAP = + ImmutableMap.of(JDBCBackendType.H2, H2Database.class.getCanonicalName()); + + // Database instance of this JDBCBackend. + private JDBCDatabase jdbcDatabase; + /** Initialize the jdbc backend instance. */ @Override public void initialize(Config config) { + jdbcDatabase = startJDBCDatabaseIfNecessary(config); + SqlSessionFactoryHelper.getInstance().init(config); SQLExceptionConverterFactory.initConverter(config); } @Override public List list( - Namespace namespace, Entity.EntityType entityType) { + Namespace namespace, Entity.EntityType entityType) throws IOException { switch (entityType) { case METALAKE: return (List) MetalakeMetaService.getInstance().listMetalakes(); @@ -79,7 +103,7 @@ public List list( } @Override - public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { + public boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws IOException { try { Entity entity = get(ident, entityType); return entity != null; @@ -90,7 +114,7 @@ public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { @Override public void insert(E e, boolean overwritten) - throws EntityAlreadyExistsException { + throws EntityAlreadyExistsException, IOException { if (e instanceof BaseMetalake) { MetalakeMetaService.getInstance().insertMetalake((BaseMetalake) e, overwritten); } else if (e instanceof CatalogEntity) { @@ -118,7 +142,7 @@ public void insert(E e, boolean overwritten) @Override public E update( NameIdentifier ident, Entity.EntityType entityType, Function updater) - throws IOException, NoSuchEntityException, AlreadyExistsException { + throws IOException, NoSuchEntityException, EntityAlreadyExistsException { switch (entityType) { case METALAKE: return (E) MetalakeMetaService.getInstance().updateMetalake(ident, updater); @@ -144,7 +168,8 @@ public E update( @Override public E get( - NameIdentifier ident, Entity.EntityType entityType) throws NoSuchEntityException { + NameIdentifier ident, Entity.EntityType entityType) + throws NoSuchEntityException, IOException { switch (entityType) { case METALAKE: return (E) MetalakeMetaService.getInstance().getMetalakeByIdentifier(ident); @@ -171,7 +196,8 @@ public E get( } @Override - public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) { + public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) + throws IOException { switch (entityType) { case METALAKE: return MetalakeMetaService.getInstance().deleteMetalake(ident, cascade); @@ -198,44 +224,46 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea } @Override - public int hardDeleteLegacyData(Entity.EntityType entityType, long legacyTimeLine) { + public int hardDeleteLegacyData(Entity.EntityType entityType, long legacyTimeline) + throws IOException { switch (entityType) { case METALAKE: return MetalakeMetaService.getInstance() - .deleteMetalakeMetasByLegacyTimeLine( - legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + .deleteMetalakeMetasByLegacyTimeline( + legacyTimeline, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); case CATALOG: return CatalogMetaService.getInstance() - .deleteCatalogMetasByLegacyTimeLine( - legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + .deleteCatalogMetasByLegacyTimeline( + legacyTimeline, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); case SCHEMA: return SchemaMetaService.getInstance() - .deleteSchemaMetasByLegacyTimeLine( - legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + .deleteSchemaMetasByLegacyTimeline( + legacyTimeline, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); case TABLE: return TableMetaService.getInstance() - .deleteTableMetasByLegacyTimeLine( - legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + .deleteTableMetasByLegacyTimeline( + legacyTimeline, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); case FILESET: return FilesetMetaService.getInstance() - .deleteFilesetAndVersionMetasByLegacyTimeLine( - legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + .deleteFilesetAndVersionMetasByLegacyTimeline( + legacyTimeline, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); case TOPIC: return TopicMetaService.getInstance() - .deleteTopicMetasByLegacyTimeLine( - legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + .deleteTopicMetasByLegacyTimeline( + legacyTimeline, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); case USER: return UserMetaService.getInstance() - .deleteUserMetasByLegacyTimeLine( - legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + .deleteUserMetasByLegacyTimeline( + legacyTimeline, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); case GROUP: return GroupMetaService.getInstance() - .deleteGroupMetasByLegacyTimeLine( - legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + .deleteGroupMetasByLegacyTimeline( + legacyTimeline, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); case ROLE: return RoleMetaService.getInstance() - .deleteRoleMetasByLegacyTimeLine( - legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + .deleteRoleMetasByLegacyTimeline( + legacyTimeline, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + case TAG: case COLUMN: case AUDIT: return 0; @@ -248,7 +276,8 @@ public int hardDeleteLegacyData(Entity.EntityType entityType, long legacyTimeLin } @Override - public int deleteOldVersionData(Entity.EntityType entityType, long versionRetentionCount) { + public int deleteOldVersionData(Entity.EntityType entityType, long versionRetentionCount) + throws IOException { switch (entityType) { case METALAKE: case CATALOG: @@ -260,6 +289,7 @@ public int deleteOldVersionData(Entity.EntityType entityType, long versionRetent case GROUP: case AUDIT: case ROLE: + case TAG: // These entity types have not implemented multi-versions, so we can skip. return 0; @@ -277,5 +307,53 @@ public int deleteOldVersionData(Entity.EntityType entityType, long versionRetent @Override public void close() throws IOException { SqlSessionFactoryHelper.getInstance().close(); + + if (jdbcDatabase != null) { + jdbcDatabase.close(); + } + } + + enum JDBCBackendType { + H2(true), + MYSQL(false); + + private final boolean embedded; + + JDBCBackendType(boolean embedded) { + this.embedded = embedded; + } + + public static JDBCBackendType fromURI(String jdbcURI) { + if (jdbcURI.startsWith("jdbc:h2")) { + return JDBCBackendType.H2; + } else if (jdbcURI.startsWith("jdbc:mysql")) { + return JDBCBackendType.MYSQL; + } else { + throw new IllegalArgumentException("Unknown JDBC URI: " + jdbcURI); + } + } + } + + /** Start JDBC database if necessary. For example, start the H2 database if the backend is H2. */ + private static JDBCDatabase startJDBCDatabaseIfNecessary(Config config) { + String jdbcUrl = config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL); + JDBCBackendType jdbcBackendType = JDBCBackendType.fromURI(jdbcUrl); + + // Not an embedded database. + if (!jdbcBackendType.embedded) { + return null; + } + + try { + JDBCDatabase jdbcDatabase = + (JDBCDatabase) + Class.forName(EMBEDDED_JDBC_DATABASE_MAP.get(jdbcBackendType)) + .getDeclaredConstructor() + .newInstance(); + jdbcDatabase.initialize(config); + return jdbcDatabase; + } catch (Exception e) { + throw new RuntimeException("Failed to create and initialize JDBCBackend.", e); + } } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCDatabase.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCDatabase.java new file mode 100644 index 00000000000..35bd2a44514 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCDatabase.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.storage.relational; + +import com.datastrato.gravitino.Config; +import java.io.Closeable; + +public interface JDBCDatabase extends Closeable { + + /** + * Initializes the Relational database environment with the provided configuration. + * + * @param config The configuration for the database backend. + * @throws RuntimeException + */ + void initialize(Config config); +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java index 17e77533e42..e365bd808e1 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational; @@ -35,9 +49,10 @@ public interface RelationalBackend extends Closeable { * if the entities does not exist. * @throws NoSuchEntityException If the corresponding parent entity of these list entities cannot * be found. + * @throws IOException If the store operation fails */ List list(Namespace namespace, Entity.EntityType entityType) - throws NoSuchEntityException; + throws NoSuchEntityException, IOException; /** * Checks the entity associated with the given identifier and entityType whether exists. @@ -45,8 +60,9 @@ List list(Namespace namespace, Entity.Enti * @param ident The identifier of the entity. * @param entityType The type of the entity. * @return True, if the entity can be found, else return false. + * @throws IOException If the store operation fails */ - boolean exists(NameIdentifier ident, Entity.EntityType entityType); + boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws IOException; /** * Stores the entity, possibly overwriting an existing entity if specified. @@ -54,9 +70,10 @@ List list(Namespace namespace, Entity.Enti * @param e The entity which need be stored. * @param overwritten If true, overwrites the existing value. * @throws EntityAlreadyExistsException If the entity already exists and overwrite is false. + * @throws IOException If the store operation fails */ void insert(E e, boolean overwritten) - throws EntityAlreadyExistsException; + throws EntityAlreadyExistsException, IOException; /** * Updates the entity. @@ -90,18 +107,21 @@ E get(NameIdentifier ident, Entity.EntityType * @param entityType The type of the entity. * @param cascade True, If you need to cascade delete entities, else false. * @return True, if the entity was successfully deleted, else false. + * @throws IOException If the store operation fails */ - boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade); + boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) + throws IOException; /** * Permanently deletes the legacy data that has been marked as deleted before the given legacy * timeline. * * @param entityType The type of the entity. - * @param legacyTimeLine The time before which the data has been marked as deleted. + * @param legacyTimeline The time before which the data has been marked as deleted. * @return The count of the deleted data. + * @throws IOException If the store operation fails */ - int hardDeleteLegacyData(Entity.EntityType entityType, long legacyTimeLine); + int hardDeleteLegacyData(Entity.EntityType entityType, long legacyTimeline) throws IOException; /** * Soft deletes the old version data that is older than or equal to the given version retention @@ -110,6 +130,8 @@ E get(NameIdentifier ident, Entity.EntityType * @param entityType The type of the entity. * @param versionRetentionCount The count of versions to retain. * @return The count of the deleted data. + * @throws IOException If the store operation fails */ - int deleteOldVersionData(Entity.EntityType entityType, long versionRetentionCount); + int deleteOldVersionData(Entity.EntityType entityType, long versionRetentionCount) + throws IOException; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java index cf8ae99458d..b205e79d19f 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational; @@ -15,7 +29,6 @@ import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.utils.Executable; import com.google.common.collect.ImmutableMap; @@ -88,7 +101,7 @@ public void put(E e, boolean overwritten) @Override public E update( NameIdentifier ident, Class type, Entity.EntityType entityType, Function updater) - throws IOException, NoSuchEntityException, AlreadyExistsException { + throws IOException, NoSuchEntityException, EntityAlreadyExistsException { return backend.update(ident, entityType, updater); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalGarbageCollector.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalGarbageCollector.java index 87e99ba39ef..44927348e4c 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalGarbageCollector.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalGarbageCollector.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational; @@ -46,11 +60,11 @@ public RelationalGarbageCollector(RelationalBackend backend, Config config) { } public void start() { - long dateTimeLineMinute = storeDeleteAfterTimeMillis / 1000 / 60; + long dateTimelineMinute = storeDeleteAfterTimeMillis / 1000 / 60; - // We will collect garbage every 10 minutes at least. If the dateTimeLineMinute is larger than - // 100 minutes, we would collect garbage every dateTimeLineMinute/10 minutes. - long frequency = Math.max(dateTimeLineMinute / 10, 10); + // We will collect garbage every 10 minutes at least. If the dateTimelineMinute is larger than + // 100 minutes, we would collect garbage every dateTimelineMinute/10 minutes. + long frequency = Math.max(dateTimelineMinute / 10, 10); garbageCollectorPool.scheduleAtFixedRate(this::collectAndClean, 5, frequency, TimeUnit.MINUTES); } @@ -60,16 +74,16 @@ private void collectAndClean() { try { LOG.info("Start to collect and delete legacy data by thread {}", threadId); - long legacyTimeLine = System.currentTimeMillis() - storeDeleteAfterTimeMillis; + long legacyTimeline = System.currentTimeMillis() - storeDeleteAfterTimeMillis; for (Entity.EntityType entityType : Entity.EntityType.values()) { long deletedCount = Long.MAX_VALUE; LOG.info( "Try to physically delete {} legacy data that has been marked deleted before {}", entityType, - legacyTimeLine); + legacyTimeline); try { while (deletedCount > 0) { - deletedCount = backend.hardDeleteLegacyData(entityType, legacyTimeLine); + deletedCount = backend.hardDeleteLegacyData(entityType, legacyTimeline); } } catch (RuntimeException e) { LOG.error("Failed to physically delete type of " + entityType + "'s legacy data: ", e); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/H2ExceptionConverter.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/H2ExceptionConverter.java index b47f8ce208d..f43a5b6b23a 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/H2ExceptionConverter.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/H2ExceptionConverter.java @@ -1,12 +1,26 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.converters; import com.datastrato.gravitino.Entity; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; -import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import com.datastrato.gravitino.EntityAlreadyExistsException; +import java.io.IOException; import java.sql.SQLException; /** @@ -19,13 +33,13 @@ public class H2ExceptionConverter implements SQLExceptionConverter { @SuppressWarnings("FormatStringAnnotation") @Override - public GravitinoRuntimeException toGravitinoException( - SQLException se, Entity.EntityType type, String name) { + public void toGravitinoException(SQLException se, Entity.EntityType type, String name) + throws IOException { switch (se.getErrorCode()) { case DUPLICATED_ENTRY_ERROR_CODE: - return new AlreadyExistsException(se, se.getMessage()); + throw new EntityAlreadyExistsException(se, se.getMessage()); default: - return new GravitinoRuntimeException(se, se.getMessage()); + throw new IOException(se); } } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/MySQLExceptionConverter.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/MySQLExceptionConverter.java index de8136629cc..320f55b1907 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/MySQLExceptionConverter.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/MySQLExceptionConverter.java @@ -1,12 +1,26 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.converters; import com.datastrato.gravitino.Entity; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; -import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import com.datastrato.gravitino.EntityAlreadyExistsException; +import java.io.IOException; import java.sql.SQLException; /** @@ -20,13 +34,13 @@ public class MySQLExceptionConverter implements SQLExceptionConverter { @SuppressWarnings("FormatStringAnnotation") @Override - public GravitinoRuntimeException toGravitinoException( - SQLException se, Entity.EntityType type, String name) { + public void toGravitinoException(SQLException se, Entity.EntityType type, String name) + throws IOException { switch (se.getErrorCode()) { case DUPLICATED_ENTRY_ERROR_CODE: - return new AlreadyExistsException(se, se.getMessage()); + throw new EntityAlreadyExistsException(se, se.getMessage()); default: - return new GravitinoRuntimeException(se, se.getMessage()); + throw new IOException(se); } } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverter.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverter.java index 47fe684cf93..081ea5b8dee 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverter.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverter.java @@ -1,11 +1,25 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.converters; import com.datastrato.gravitino.Entity; -import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import java.io.IOException; import java.sql.SQLException; /** Interface for converter JDBC SQL exceptions to Gravitino exceptions. */ @@ -16,9 +30,7 @@ public interface SQLExceptionConverter { * @param sqlException The sql exception to map * @param type The type of the entity * @param name The name of the entity - * @return A best attempt at a corresponding jdbc connector exception or generic with the - * SQLException as the cause */ - GravitinoRuntimeException toGravitinoException( - SQLException sqlException, Entity.EntityType type, String name); + void toGravitinoException(SQLException sqlException, Entity.EntityType type, String name) + throws IOException; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java index 3623b3dc3e6..1bf1ee3a627 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.converters; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/database/H2Database.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/database/H2Database.java new file mode 100644 index 00000000000..19edb7aeee0 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/database/H2Database.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.storage.relational.database; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Configs; +import com.datastrato.gravitino.storage.relational.JDBCDatabase; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class H2Database implements JDBCDatabase { + private static final Logger LOG = LoggerFactory.getLogger(H2Database.class); + private String h2ConnectionUri; + private String username; + private String password; + + @Override + public void initialize(Config config) { + this.h2ConnectionUri = startH2Database(config); + } + + public String startH2Database(Config config) { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String storagePath = getStoragePath(config); + String originalJDBCUrl = config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL); + this.username = config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER); + this.password = config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD); + + String connectionUrl = constructH2URI(originalJDBCUrl, storagePath); + + try (Connection connection = DriverManager.getConnection(connectionUrl, username, password); + Statement statement = connection.createStatement()) { + String sqlContent = + FileUtils.readFileToString( + new File(gravitinoHome + "/scripts/h2/schema-h2.sql"), StandardCharsets.UTF_8); + + statement.execute(sqlContent); + } catch (Exception e) { + LOG.error("Failed to create table for H2 database.", e); + throw new RuntimeException("Failed to create table for H2 database.", e); + } + + config.set(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL, connectionUrl); + + return connectionUrl; + } + + private static String constructH2URI(String originURI, String storagePath) { + if (!originURI.contains(":file:")) { + originURI = "jdbc:h2:file:" + storagePath; + } + + if (!originURI.contains("DB_CLOSE_DELAY")) { + originURI = originURI + ";DB_CLOSE_DELAY=-1"; + } + + if (!originURI.contains("MODE")) { + originURI = originURI + ";MODE=MYSQL"; + } + + return originURI; + } + + private static String getStoragePath(Config config) { + String dbPath = config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PATH); + if (StringUtils.isBlank(dbPath)) { + return Configs.DEFAULT_RELATIONAL_JDBC_BACKEND_PATH; + } + + Path path = Paths.get(dbPath); + // Relative Path + if (!path.isAbsolute()) { + path = Paths.get(System.getenv("GRAVITINO_HOME"), dbPath); + return path.toString(); + } + + return dbPath; + } + + @Override + public void close() throws IOException { + try (Connection connection = DriverManager.getConnection(h2ConnectionUri, username, password); + Statement statement = connection.createStatement()) { + statement.execute("SHUTDOWN"); + } catch (Exception e) { + LOG.error("Failed to shutdown H2 database.", e); + throw new RuntimeException("Failed to shutdown H2 database.", e); + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java index 5e40deb39b1..7356cb5947e 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.mapper; @@ -54,6 +68,17 @@ Long selectCatalogIdByMetalakeIdAndName( CatalogPO selectCatalogMetaByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("catalogName") String name); + @Select( + "SELECT catalog_id as catalogId, catalog_name as catalogName," + + " metalake_id as metalakeId, type, provider," + + " catalog_comment as catalogComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + CatalogPO selectCatalogMetaById(@Param("catalogId") Long catalogId); + @Insert( "INSERT INTO " + TABLE_NAME @@ -138,21 +163,23 @@ Integer updateCatalogMeta( @Update( "UPDATE " + TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") Integer softDeleteCatalogMetasByCatalogId(@Param("catalogId") Long catalogId); @Update( "UPDATE " + TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") Integer softDeleteCatalogMetasByMetalakeId(@Param("metalakeId") Long metalakeId); @Delete( "DELETE FROM " + TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") - Integer deleteCatalogMetasByLegacyTimeLine( - @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + Integer deleteCatalogMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetMetaMapper.java index ee87daa43fd..c3c6cff275c 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetMetaMapper.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.mapper; @@ -112,6 +126,44 @@ Long selectFilesetIdBySchemaIdAndName( FilesetPO selectFilesetMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("filesetName") String name); + @Select( + "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," + + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," + + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," + + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," + + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," + + " vi.deleted_at as version_deleted_at" + + " FROM " + + META_TABLE_NAME + + " fm INNER JOIN " + + VERSION_TABLE_NAME + + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" + + " WHERE fm.fileset_id = #{filesetId}" + + " AND fm.deleted_at = 0 AND vi.deleted_at = 0") + @Results({ + @Result(property = "filesetId", column = "fileset_id"), + @Result(property = "filesetName", column = "fileset_name"), + @Result(property = "metalakeId", column = "metalake_id"), + @Result(property = "catalogId", column = "catalog_id"), + @Result(property = "schemaId", column = "schema_id"), + @Result(property = "type", column = "type"), + @Result(property = "auditInfo", column = "audit_info"), + @Result(property = "currentVersion", column = "current_version"), + @Result(property = "lastVersion", column = "last_version"), + @Result(property = "deletedAt", column = "deleted_at"), + @Result(property = "filesetVersionPO.id", column = "id"), + @Result(property = "filesetVersionPO.metalakeId", column = "version_metalake_id"), + @Result(property = "filesetVersionPO.catalogId", column = "version_catalog_id"), + @Result(property = "filesetVersionPO.schemaId", column = "version_schema_id"), + @Result(property = "filesetVersionPO.filesetId", column = "version_fileset_id"), + @Result(property = "filesetVersionPO.version", column = "version"), + @Result(property = "filesetVersionPO.filesetComment", column = "fileset_comment"), + @Result(property = "filesetVersionPO.properties", column = "properties"), + @Result(property = "filesetVersionPO.storageLocation", column = "storage_location"), + @Result(property = "filesetVersionPO.deletedAt", column = "version_deleted_at") + }) + FilesetPO selectFilesetMetaById(@Param("filesetId") Long filesetId); + @Insert( "INSERT INTO " + META_TABLE_NAME @@ -191,35 +243,39 @@ Integer updateFilesetMeta( @Update( "UPDATE " + META_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") Integer softDeleteFilesetMetasByMetalakeId(@Param("metalakeId") Long metalakeId); @Update( "UPDATE " + META_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") Integer softDeleteFilesetMetasByCatalogId(@Param("catalogId") Long catalogId); @Update( "UPDATE " + META_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE schema_id = #{schemaId} AND deleted_at = 0") Integer softDeleteFilesetMetasBySchemaId(@Param("schemaId") Long schemaId); @Update( "UPDATE " + META_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE fileset_id = #{filesetId} AND deleted_at = 0") Integer softDeleteFilesetMetasByFilesetId(@Param("filesetId") Long filesetId); @Delete( "DELETE FROM " + META_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") - Integer deleteFilesetMetasByLegacyTimeLine( - @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + Integer deleteFilesetMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetVersionMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetVersionMapper.java index a16ee520916..020839b91b1 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetVersionMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetVersionMapper.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.mapper; @@ -77,37 +91,41 @@ void insertFilesetVersionOnDuplicateKeyUpdate( @Update( "UPDATE " + VERSION_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") Integer softDeleteFilesetVersionsByMetalakeId(@Param("metalakeId") Long metalakeId); @Update( "UPDATE " + VERSION_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") Integer softDeleteFilesetVersionsByCatalogId(@Param("catalogId") Long catalogId); @Update( "UPDATE " + VERSION_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE schema_id = #{schemaId} AND deleted_at = 0") Integer softDeleteFilesetVersionsBySchemaId(@Param("schemaId") Long schemaId); @Update( "UPDATE " + VERSION_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE fileset_id = #{filesetId} AND deleted_at = 0") Integer softDeleteFilesetVersionsByFilesetId(@Param("filesetId") Long filesetId); @Delete( "DELETE FROM " + VERSION_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") - Integer deleteFilesetVersionsByLegacyTimeLine( - @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + Integer deleteFilesetVersionsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); @Select( "SELECT fileset_id as filesetId," @@ -122,7 +140,8 @@ List selectFilesetVersionsByRetentionCount( @Update( "UPDATE " + VERSION_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE fileset_id = #{filesetId} AND version <= #{versionRetentionLine} AND deleted_at = 0 LIMIT #{limit}") Integer softDeleteFilesetVersionsByRetentionLine( @Param("filesetId") Long filesetId, diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupMetaMapper.java index 2ee109f51f8..3c051e169be 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupMetaMapper.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.mapper; @@ -90,14 +104,16 @@ GroupPO selectGroupMetaByMetalakeIdAndName( @Update( "UPDATE " + GROUP_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE group_id = #{groupId} AND deleted_at = 0") void softDeleteGroupMetaByGroupId(@Param("groupId") Long groupId); @Update( "UPDATE " + GROUP_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") void softDeleteGroupMetasByMetalakeId(@Param("metalakeId") Long metalakeId); @@ -137,7 +153,7 @@ Integer updateGroupMeta( @Delete( "DELETE FROM " + GROUP_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") - Integer deleteGroupMetasByLegacyTimeLine( - @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + Integer deleteGroupMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupRoleRelMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupRoleRelMapper.java index 4297a937813..e8967213142 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupRoleRelMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupRoleRelMapper.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.mapper; @@ -75,7 +89,8 @@ void batchInsertGroupRoleRelOnDuplicateKeyUpdate( @Update( "UPDATE " + GROUP_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE group_id = #{groupId} AND deleted_at = 0") void softDeleteGroupRoleRelByGroupId(@Param("groupId") Long groupId); @@ -83,7 +98,8 @@ void batchInsertGroupRoleRelOnDuplicateKeyUpdate( "" + }) + void batchInsertSecurableObjects( + @Param("securableObjects") List securableObjectPOs); + + @Update( + "UPDATE " + + SECURABLE_OBJECT_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE role_id = #{roleId} AND deleted_at = 0") + void softDeleteSecurableObjectsByRoleId(@Param("roleId") Long roleId); + + @Update( + "UPDATE " + + SECURABLE_OBJECT_TABLE_NAME + + " ob SET ob.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " where exists ( select * from " + + ROLE_TABLE_NAME + + " ro WHERE ro.metalake_id = #{metalakeId} AND ro.role_id = ob.role_id" + + " AND ro.deleted_at = 0) AND ob.deleted_at = 0") + void softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId); + + @Select( + "SELECT role_id as roleId, entity_id as entityId," + + " type as type, privilege_names as privilegeNames," + + " privilege_conditions as privilegeConditions, current_version as currentVersion," + + " last_version as lastVersion, deleted_at as deletedAt" + + " FROM " + + SECURABLE_OBJECT_TABLE_NAME + + " WHERE role_id = #{roleId} AND deleted_at = 0") + List listSecurableObjectsByRoleId(@Param("roleId") Long roleId); + + @Delete( + "DELETE FROM " + + SECURABLE_OBJECT_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + Integer deleteSecurableObjectsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TableMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TableMetaMapper.java index aeeaa1eac14..1dd3830868f 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TableMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TableMetaMapper.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.mapper; @@ -55,6 +69,17 @@ Long selectTableIdBySchemaIdAndName( TablePO selectTableMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("tableName") String name); + @Select( + "SELECT table_id as tableId, table_name as tableName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_id as schemaId, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE table_id = #{tableId} AND deleted_at = 0") + TablePO selectTableMetaById(@Param("tableId") Long tableId); + @Insert( "INSERT INTO " + TABLE_NAME @@ -128,35 +153,39 @@ Integer updateTableMeta( @Update( "UPDATE " + TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE table_id = #{tableId} AND deleted_at = 0") Integer softDeleteTableMetasByTableId(@Param("tableId") Long tableId); @Update( "UPDATE " + TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") Integer softDeleteTableMetasByMetalakeId(@Param("metalakeId") Long metalakeId); @Update( "UPDATE " + TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") Integer softDeleteTableMetasByCatalogId(@Param("catalogId") Long catalogId); @Update( "UPDATE " + TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE schema_id = #{schemaId} AND deleted_at = 0") Integer softDeleteTableMetasBySchemaId(@Param("schemaId") Long schemaId); @Delete( "DELETE FROM " + TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") - Integer deleteTableMetasByLegacyTimeLine( - @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + Integer deleteTableMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TopicMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TopicMetaMapper.java index 06f0d611db0..41371b6c6a3 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TopicMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TopicMetaMapper.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.mapper; @@ -91,6 +105,17 @@ public interface TopicMetaMapper { TopicPO selectTopicMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("topicName") String topicName); + @Select( + "SELECT topic_id as topicId, topic_name as topicName," + + " metalake_id as metalakeId, catalog_id as catalogId, schema_id as schemaId," + + " comment as comment, properties as properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE topic_id = #{topicId} AND deleted_at = 0") + TopicPO selectTopicMetaById(@Param("topicId") Long topicId); + @Update( "UPDATE " + TABLE_NAME @@ -129,35 +154,39 @@ Long selectTopicIdBySchemaIdAndName( @Update( "UPDATE " + TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE topic_id = #{topicId} AND deleted_at = 0") Integer softDeleteTopicMetasByTopicId(@Param("topicId") Long topicId); @Update( "UPDATE " + TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") Integer softDeleteTopicMetasByCatalogId(@Param("catalogId") Long catalogId); @Update( "UPDATE " + TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") Integer softDeleteTopicMetasByMetalakeId(@Param("metalakeId") Long metalakeId); @Update( "UPDATE " + TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE schema_id = #{schemaId} AND deleted_at = 0") Integer softDeleteTopicMetasBySchemaId(@Param("schemaId") Long schemaId); @Delete( "DELETE FROM " + TABLE_NAME - + " WHERE deleted_at != 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") - Integer deleteTopicMetasByLegacyTimeLine( - @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); + + " WHERE deleted_at != 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + Integer deleteTopicMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserMetaMapper.java index 69df436a481..6ffe2c57e48 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserMetaMapper.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.mapper; @@ -90,14 +104,16 @@ UserPO selectUserMetaByMetalakeIdAndName( @Update( "UPDATE " + USER_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE user_id = #{userId} AND deleted_at = 0") void softDeleteUserMetaByUserId(@Param("userId") Long userId); @Update( "UPDATE " + USER_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") void softDeleteUserMetasByMetalakeId(@Param("metalakeId") Long metalakeId); @@ -137,7 +153,7 @@ Integer updateUserMeta( @Delete( "DELETE FROM " + USER_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") - Integer deleteUserMetasByLegacyTimeLine( - @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + Integer deleteUserMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserRoleRelMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserRoleRelMapper.java index 3135476f55a..bdf2735bb0e 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserRoleRelMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserRoleRelMapper.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.storage.relational.mapper; @@ -75,7 +89,8 @@ void batchInsertUserRoleRelOnDuplicateKeyUpdate( @Update( "UPDATE " + USER_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " + + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE user_id = #{userId} AND deleted_at = 0") void softDeleteUserRoleRelByUserId(@Param("userId") Long userId); @@ -83,7 +98,8 @@ void batchInsertUserRoleRelOnDuplicateKeyUpdate( "