[apache#2756] feat(PyClient): Integration test with Gravitino Python …

…client (apache#2797)

### What changes were proposed in this pull request?

Add two Gradle command in the client-python module
1. `./gradlew :clients:client-python:test`
2. `./gradlew :clients:client-python: integrationTest`

### Why are the changes needed?

Proved a real test environment for Gravitino server. 

Fix: apache#2756

### Does this PR introduce _any_ user-facing change?

N/A

### How was this patch tested?

CI Passed

.github/workflows/backend-integration-test.yml

@@ -91,7 +91,7 @@ jobs:
       - name: Backend Integration Test
         id: integrationTest
         run: |
-          ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }} -PskipPythonITs
+          ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }}
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

.github/workflows/python-integration-test.yml

@@ -64,9 +64,10 @@ jobs:
       - name: Python Client Integration Test
         id: integrationTest
         run: |
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
           for pythonVersion in "3.8" "3.9" "3.10" "3.11"
           do
-            ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} :client:client-python:test
+            ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} :client:client-python:integrationTest
           done
 
       - name: Upload integrate tests reports

clients/client-python/README.md

@@ -15,4 +15,17 @@
 1. Install dependency
     ```bash
     pip install -e .[dev]
-    ```
+    ```
+
+2. Run tests
+    ```bash
+    cd gravitino
+    ./gradlew :clients:client-python:test
+    ```
+
+3. Run integration tests
+    ```bash
+    cd gravitino
+    ./gradlew compileDistribution -x test
+    ./gradlew :clients:client-python:integrationTest
+    ```

clients/client-python/build.gradle.kts

@@ -24,7 +24,7 @@ fun deleteCacheDir(targetDir: String) {
 tasks {
   val pipInstall by registering(VenvTask::class) {
     venvExec = "pip"
-    args = listOf("install", "-e", ".")
+    args = listOf("install", "-e", ".[dev]")
   }
 
   val test by registering(VenvTask::class) {
@@ -34,6 +34,15 @@ tasks {
     workingDir = projectDir.resolve(".")
   }
 
+  val integrationTest by registering(VenvTask::class) {
+    dependsOn(pipInstall)
+    venvExec = "python"
+    args = listOf("-m", "unittest", "tests/test_integration_gravitino_client.py")
+    workingDir = projectDir.resolve(".")
+    environment = mapOf("PROJECT_VERSION" to project.version,
+      "GRAVITINO_HOME" to project.rootDir.path + "/distribution/package")
+  }
+
   val build by registering(VenvTask::class) {
   }
 

clients/client-python/requirements-dev.txt

@@ -1,2 +1,3 @@
 # Copyright 2024 Datastrato Pvt Ltd.
-# This software is licensed under the Apache License version 2.
+# This software is licensed under the Apache License version 2.
+requests

clients/client-python/tests/integration_test_env.py

@@ -0,0 +1,101 @@
+"""
+Copyright 2024 Datastrato Pvt Ltd.
+This software is licensed under the Apache License version 2.
+"""
+import logging
+import os
+import unittest
+import subprocess
+import time
+import requests
+
+from gravitino import GravitinoClient
+from gravitino.constants import TIMEOUT
+
+logger = logging.getLogger(__name__)
+
+
+def check_gravitino_server_status():
+    try:
+        response = requests.get("http://localhost:8090/api/version")
+        response.raise_for_status()  # raise an exception for bad status codes
+        response.close()
+        return True
+    except requests.exceptions.RequestException as e:
+        logger.error("Failed to access the server: {}", e)
+        return False
+
+
+def _init_logging():
+    logging.basicConfig(level=logging.DEBUG)
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.DEBUG)
+    logger.addHandler(console_handler)
+
+
+# Provide real test environment for the Gravitino Server
+class IntegrationTestEnv(unittest.TestCase):
+    GravitinoHome = None
+
+    @classmethod
+    def setUpClass(cls):
+        _init_logging()
+        logger.info("Starting integration test environment...")
+        cls.GravitinoHome = os.environ.get('GRAVITINO_HOME')
+        if cls.GravitinoHome is None:
+            logger.warning('WARN: Currently, Python Client integration test only runs in the Gradle build environment, '
+                  'Please execute `./gradlew :clients:client-python:test` in the gravitino project root directory.')
+            quit(0)
+
+        # Start Gravitino Server
+        result = subprocess.run([cls.GravitinoHome + '/bin/gravitino.sh', 'start'], capture_output=True, text=True)
+        if result.stdout:
+            logger.info("stdout: %s", result.stdout)
+        if result.stderr:
+            logger.info("stderr: %s", result.stderr)
+
+        gravitinoServerRunning = False
+        for i in range(5):
+            logger.info("Monitoring Gravitino server status. Attempt %s", i + 1)
+            if check_gravitino_server_status():
+                logger.debug("Gravitino Server is running")
+                gravitinoServerRunning = True
+                break
+            else:
+                logger.debug("Gravitino Server is not running")
+                time.sleep(1)
+
+        if not gravitinoServerRunning:
+            logger.error("ERROR: Can't start Gravitino server!")
+            quit(0)
+
+        cls.client = GravitinoClient("http://localhost:8090", timeout=TIMEOUT)
+
+    @classmethod
+    def tearDownClass(cls):
+        logger.info("Stop integration test environment...")
+        result = subprocess.run([cls.GravitinoHome + '/bin/gravitino.sh', 'stop'], capture_output=True, text=True)
+        if result.stdout:
+            logger.info("stdout: %s", result.stdout)
+        if result.stderr:
+            logger.info("stderr: %s", result.stderr)
+
+        gravitinoServerRunning = True
+        for i in range(5):
+            logger.debug("Monitoring Gravitino server status. Attempt %s", i + 1)
+            if check_gravitino_server_status():
+                logger.debug("Gravitino server still running")
+                time.sleep(1)
+            else:
+                logger.debug("Stop Gravitino server successes!")
+                gravitinoServerRunning = False
+                break
+
+        if gravitinoServerRunning:
+            logger.error("ERROR: Can't stop Gravitino server!")
+
+    # Determine whether to run from Gradle base on environment variables
+    # integrated test environment (ITE)
+    @staticmethod
+    def not_in_ITE():
+        return os.environ.get('GRAVITINO_HOME') is None and os.environ.get('PROJECT_VERSION') is None

clients/client-python/tests/test_integration_gravitino_client.py

@@ -0,0 +1,23 @@
+"""
+Copyright 2024 Datastrato Pvt Ltd.
+This software is licensed under the Apache License version 2.
+"""
+
+import os
+import unittest
+from tests.integration_test_env import IntegrationTestEnv
+
+
+@unittest.skipIf(IntegrationTestEnv.not_in_ITE(),
+                 "Currently, Python Client integration test only runs in the Gradle build environment")
+class IntegrationTestGravitinoClient(IntegrationTestEnv):
+    def test_version(self):
+        versionDTO = self.client.version
+        assert versionDTO['version'] is not None
+
+        # Get project version from environment (Setting by Gradle build script `build.gradle.kts`),
+        # But if you directly execute this test in IDEA, maybe can not get it.
+        projectVersion = os.environ.get('PROJECT_VERSION', '')
+
+        if projectVersion != '':
+            assert versionDTO['version'] == projectVersion

docs/how-to-test.md

@@ -72,7 +72,6 @@ To deploy the Gravitino server locally to run the integration tests, follow thes
 * Skip unit tests by using the `./gradlew build -PskipTests` command.
 * Skip integration tests by using the `./gradlew build -PskipITs` command.
 * Skip web frontend integration tests by using the `./gradlew build -PskipWebITs` command.
-* Skip Python client integration tests by using the `./gradlew build -PskipPythonITs` command.
 * Skip both unit tests and integration tests by using the `./gradlew build -x test` or `./gradlew build -PskipTests -PskipITs` commands.
 
 ## Configuring parameters for integration tests