[apache#2756] Integration test with Gravitino Python client

.github/workflows/backend-integration-test.yml

@@ -91,7 +91,7 @@ jobs:
       - name: Backend Integration Test
         id: integrationTest
         run: |
-          ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }} -PskipPythonITs
+          ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }}
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

.github/workflows/frontend-integration-test.yml

@@ -59,7 +59,7 @@ jobs:
     strategy:
       matrix:
         architecture: [linux/amd64]
-        java-version: [ 8, 11, 17 ]
+        java-version: [ 8 ]
         test-mode: [ embedded, deploy ]
     env:
       PLATFORM: ${{ matrix.architecture }}

.github/workflows/python-integration-test.yml

@@ -66,7 +66,7 @@ jobs:
         run: |
           for pythonVersion in "3.8" "3.9" "3.10" "3.11"
           do
-            ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} :client:client-python:test
+            ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} :client:client-python:integrationTest
           done
 
       - name: Upload integrate tests reports

clients/client-python/README.md

@@ -15,4 +15,16 @@
 1. Install dependency
     ```bash
     pip install -e .[dev]
-    ```
+    ```
+
+2. Run tests
+    ```bash
+    cd gravitino
+    ./gradlew :clients:client-python:test
+    ```
+
+3. Run integration tests
+    ```bash
+    cd gravitino
+    ./gradlew :clients:client-python:integrationTest
+    ```

clients/client-python/build.gradle.kts

@@ -34,6 +34,15 @@ tasks {
     workingDir = projectDir.resolve(".")
   }
 
+  val integrationTest by registering(VenvTask::class) {
+    dependsOn(pipInstall, project.rootProject.tasks.findByPath("compileDistribution"))
+    venvExec = "python"
+    args = listOf("-m", "unittest", "tests/test_integration_gravitino_client.py")
+    workingDir = projectDir.resolve(".")
+    environment = mapOf("PROJECT_VERSION" to project.version,
+      "GRAVITINO_HOME" to project.rootDir.path + "/distribution/package")
+  }
+
   val build by registering(VenvTask::class) {
   }
 

clients/client-python/tests/integration_test_env.py

@@ -0,0 +1,46 @@
+"""
+Copyright 2024 Datastrato Pvt Ltd.
+This software is licensed under the Apache License version 2.
+"""
+import os
+import unittest
+import subprocess
+import time
+
+from gravitino import GravitinoClient
+from gravitino.constants import TIMEOUT
+
+# Provide real test environment for the Gravitino Server
+class IntegrationTestEnv(unittest.TestCase):
+    GravitinoHome = None
+
+    @classmethod
+    def setUpClass(cls):
+        print("Starting integration test environment...")
+        cls.GravitinoHome = os.environ.get('GRAVITINO_HOME')
+        if cls.GravitinoHome is None:
+            print('WARN: Currently, Python Client integration test only runs in the Gradle build environment, '
+                  'Please execute `./gradlew :clients:client-python:test` in the gravitino project root directory.')
+            quit(0)
+
+        # Start Gravitino Server
+        result = subprocess.run([cls.GravitinoHome + '/bin/gravitino.sh', 'start'], capture_output=True, text=True)
+        print('stdout:', result.stdout)
+        print('stderr:', result.stderr)
+        time.sleep(3)
+        cls.client = GravitinoClient("http://localhost:8090", timeout=TIMEOUT)
+
+    @classmethod
+    def tearDownClass(cls):
+        print("Stop integration test environment...")
+        result = subprocess.run([cls.GravitinoHome + '/bin/gravitino.sh', 'stop'], capture_output=True, text=True)
+        print('stdout:', result.stdout)
+        print('stderr:', result.stderr)
+        time.sleep(3)
+
+    # Determine whether to run from Gradle base on environment variables
+    # integrated test environment (ITE)
+    @staticmethod
+    def notInITE():
+        return os.environ.get('GRAVITINO_HOME') is None and os.environ.get('PROJECT_VERSION') is None
+

clients/client-python/tests/test_integration_gravitino_client.py

@@ -0,0 +1,25 @@
+"""
+Copyright 2024 Datastrato Pvt Ltd.
+This software is licensed under the Apache License version 2.
+"""
+
+import os
+import unittest
+from tests.integration_test_env import IntegrationTestEnv
+
+
+@unittest.skipIf(IntegrationTestEnv.notInITE(),
+                 "Currently, Python Client integration test only runs in the Gradle build environment")
+class IntegrationTestGravitinoClient(IntegrationTestEnv):
+    def test_version(self):
+        versionDTO = self.client.version
+        assert versionDTO['version'] is not None
+
+        # Get project version from environment (Setting by Gradle build script `build.gradle.kts`),
+        # But if you directly execute this test in IDEA, maybe can not get it.
+        projectVersion = os.environ.get('PROJECT_VERSION', '')
+
+        if projectVersion != '':
+            print(versionDTO['version'])
+            print(projectVersion)
+            assert versionDTO['version'] == projectVersion

docs/how-to-test.md

@@ -72,7 +72,6 @@ To deploy the Gravitino server locally to run the integration tests, follow thes
 * Skip unit tests by using the `./gradlew build -PskipTests` command.
 * Skip integration tests by using the `./gradlew build -PskipITs` command.
 * Skip web frontend integration tests by using the `./gradlew build -PskipWebITs` command.
-* Skip Python client integration tests by using the `./gradlew build -PskipPythonITs` command.
 * Skip both unit tests and integration tests by using the `./gradlew build -x test` or `./gradlew build -PskipTests -PskipITs` commands.
 
 ## Configuring parameters for integration tests