[apache#4662] improve(IT): Add ranger authorization Hive E2E test (ap…

…ache#4651) ### What changes were proposed in this pull request? 1. Use an isolated class loader to create an authorization Ranger plugin. 2. Add authorization for Hive E2E tests using Ranger via Gravitino RESTful access control. 3. Copy the authorization ranger jar to the `distribution/package/authorizations/ranger/libs`. 4. Add `Apache Ranger` to license.bin ### Why are the changes needed? apache#4662 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? CI --------- Co-authored-by: yuqi <[email protected]> Co-authored-by: Jerry Shao <[email protected]>
mchades · Aug 26, 2024 · 3795285 · 3795285
1 parent ee23854
commit 3795285
Show file tree

Hide file tree

Showing 16 changed files with 782 additions and 242 deletions.
diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml
@@ -0,0 +1,96 @@
+name: Access Control Integration Test
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the "main" branch
+  push:
+    branches: [ "main", "branch-*" ]
+  pull_request:
+    branches: [ "main", "branch-*" ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            source_changes:
+              - api/**
+              - authorizations/**
+              - catalogs/**
+              - clients/client-java/**
+              - clients/client-java-runtime/**
+              - common/**
+              - core/**
+              - integration-test-common/**
+              - server/**
+              - server-common/**
+    outputs:
+      source_changes: ${{ steps.filter.outputs.source_changes }}
+
+  # Integration test for AMD64 architecture
+  test-amd64-arch:
+    needs: changes
+    if: needs.changes.outputs.source_changes == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        # Integration test for AMD64 architecture
+        architecture: [linux/amd64]
+        java-version: [ 17 ]
+        test-mode: [ embedded, deploy ]
+        include:
+          - test-mode: 'embedded'
+            backend: 'h2'
+          - test-mode: 'deploy'
+            backend: 'mysql'
+
+    env:
+      PLATFORM: ${{ matrix.architecture }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Check required command
+        run: |
+          dev/ci/check_commands.sh
+
+      - name: Package Gravitino
+        if : ${{ matrix.test-mode == 'deploy' }}
+        run: |
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
+
+      - name: Free up disk space
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Authorization Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
+        id: integrationTest
+        run: |
+          ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdbcBackend=${{ matrix.backend }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**"
+
+      - name: Upload integrate tests reports
+        uses: actions/upload-artifact@v3
+        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        with:
+          name: authorizations-integrate-test-reports-${{ matrix.java-version }}
+          path: |
+            build/reports
+            distribution/package/logs/gravitino-server.out
+            distribution/package/logs/gravitino-server.log
diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
@@ -95,13 +95,14 @@ jobs:
         run: |
           dev/ci/util_free_space.sh
 
-      - name: Backend Integration Test(JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
+      - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
         id: integrationTest
         run: >
           ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
           -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test 
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
+          -x :authorizations:authorization-ranger:test
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3
@@ -164,13 +165,14 @@ jobs:
         run: |
           dev/ci/util_free_space.sh
 
-      - name: Backend Integration Test(JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
+      - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
         id: integrationTest
         run: >
           ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
           -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test 
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
+          -x :authorizations:authorization-ranger:test
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

diff --git a/LICENSE.bin b/LICENSE.bin
@@ -310,6 +310,8 @@
    Apache Yetus - Audience Annotations
    Apache Kerby
    Apache Kyuubi
+   Apache Ranger
+   Apache Ranger intg
    Jackson JSON processor
    DataNucleus
    Modernizer Maven Plugin
@@ -364,7 +366,7 @@
 
    This product bundles various third-party components also under the
    BSD license
-   
+
    JSR305
    LevelDB JNI
    RocksDB JNI
@@ -446,6 +448,7 @@
    Jakarta RESTful WS API
    Jakarta XML Binding API
    JavaServer Pages(TM) API
+   Javax WS RS API
    HK2 API Module
    HK2 Service Locator
    HK2 Utils

diff --git a/NOTICE.bin b/NOTICE.bin
@@ -22,6 +22,9 @@ Copyright 2008-2023 The Apache Software Foundation
 Apache Zeppelin
 Copyright 2016-2023 The Apache Software Foundation
 
+Apache Ranger
+Copyright 2014-2024 The Apache Software Foundation
+
 Apache Hadoop
 Copyright 2006 and onwards The Apache Software Foundation.
 

diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts
@@ -51,11 +51,17 @@ dependencies {
     exclude("org.apache.ranger", "ranger-plugins-audit")
     exclude("org.apache.ranger", "ranger-plugins-cred")
     exclude("org.apache.ranger", "ranger-plugins-classloader")
+    exclude("javax.ws.rs")
   }
+  implementation(libs.javax.ws.rs.api)
   implementation(libs.javax.jaxb.api) {
     exclude("*")
   }
 
+  testImplementation(project(":common"))
+  testImplementation(project(":clients:client-java"))
+  testImplementation(project(":server"))
+  testImplementation(project(":catalogs:catalog-common"))
   testImplementation(project(":integration-test-common", "testArtifacts"))
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.mockito.core)
@@ -69,24 +75,33 @@ dependencies {
     exclude("org.elasticsearch")
     exclude("org.elasticsearch.client")
     exclude("org.elasticsearch.plugin")
+    exclude("javax.ws.rs")
   }
   testImplementation(libs.hive2.jdbc) {
     exclude("org.slf4j")
   }
+  testImplementation(libs.mysql.driver)
 }
 
 tasks {
   val runtimeJars by registering(Copy::class) {
     from(configurations.runtimeClasspath)
     into("build/libs")
   }
-}
 
-tasks.build {
-  dependsOn("runtimeJars", "javadoc")
+  val copyAuthorizationLibs by registering(Copy::class) {
+    dependsOn("jar", "runtimeJars")
+    from("build/libs")
+    into("$rootDir/distribution/package/authorizations/ranger/libs")
+  }
+
+  register("copyLibAndConfig", Copy::class) {
+    dependsOn(copyAuthorizationLibs)
+  }
 }
 
 tasks.test {
+  dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars")
   val skipUTs = project.hasProperty("skipTests")
   if (skipUTs) {
     // Only run integration tests
@@ -101,7 +116,3 @@ tasks.test {
     dependsOn(tasks.jar)
   }
 }
-
-tasks.getByName("generateMetadataFileForMavenJavaPublication") {
-  dependsOn("runtimeJars")
-}