Split to 2 packages (#1223)

Signed-off-by: yhmo <[email protected]>

.github/workflows/java_sdk_ci_test.yaml

@@ -31,13 +31,13 @@ jobs:
           mvn clean versions:set -DnewVersion=2.4.0
           mvn clean install -Dmaven.test.skip=true
 
-      - name: Test
-        timeout-minutes: 60
-        shell: bash
-        working-directory: tests/milvustestv2
-        run: |
-          echo "run testcases"
-          mvn clean test -Dsurefire.suiteXmlFiles=testng.xml
+#      - name: Test
+#        timeout-minutes: 60
+#        shell: bash
+#        working-directory: tests/milvustestv2
+#        run: |
+#          echo "run testcases"
+#          mvn clean test -Dsurefire.suiteXmlFiles=testng.xml
 
 #      - name: Upload logs
 #        if: ${{ always() }}

.gitignore

@@ -30,6 +30,7 @@ hs_err_pid*
 target/
 volumes/
 *.iml
+.flattened-pom.xml
 
 # Example files
 examples/bulk_writer

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "src/main/milvus-proto"]
-	path = src/main/milvus-proto
+[submodule "sdk-core/src/main/milvus-proto"]
+	path = sdk-core/src/main/milvus-proto
 	url = https://github.com/milvus-io/milvus-proto.git

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## milvus-sdk-java 2.5.2 (TBD)
+### Improvement
+- Split milvus Java SDK to two packages to reduce dependency complexity
+
 ## milvus-sdk-java 2.5.1 (2024-12-04)
 ### Improvement
 - Support upsert items with auto-id primary key

DEVELOPMENT.md

@@ -13,7 +13,7 @@ This document will help to setup your development environment and running tests
 $ git clone --recursive [email protected]:milvus-io/milvus-sdk-java.git
 ```
 
-Milvus proto files are managed by a submodule project under the directory: src/milvus-proto
+Milvus proto files are managed by a submodule project under the directory: sdk-core/src/main/milvus-proto
 Fetch Milvus proto files by the following command(If the previous clone is not with submodules)
 ```shell
 $ git submodule update --init
@@ -31,10 +31,10 @@ $  mvn install
 ```
 
 ## Update Milvus proto files
-Milvus proto files are managed by a submodule project under the directory: src/milvus-proto
+Milvus proto files are managed by a submodule project under the directory: sdk-core/src/main/milvus-proto
 Before developing new interfaces, you need to get the latest proto files by the following command:
 ```shell
-$  git submodule update --remote
+$ git submodule update --remote
 ```
 
 ## Building Milvus

README.md

@@ -21,7 +21,7 @@ The following table shows compatibilities between Milvus and Java SDK.
 |    >= 2.2.9    |  2.2.7 ~ 2.2.15  |
 |     2.3.x      |      2.3.11      |
 |     2.4.x      |      2.4.8       |
-|     2.5.x      |      2.5.1       |
+|     2.5.x      |      2.5.2       |
 
 ### Install Java SDK
 
@@ -33,22 +33,47 @@ You can use **Apache Maven** or **Gradle** add Milvus SDK to your project.
         <dependency>
             <groupId>io.milvus</groupId>
             <artifactId>milvus-sdk-java</artifactId>
-            <version>2.5.1</version>
+            <version>2.5.2</version>
         </dependency>
        ```
 
    - Gradle/Groovy
 
         ```groovy
-        implementation 'io.milvus:milvus-sdk-java:2.5.1'
+        implementation 'io.milvus:milvus-sdk-java:2.5.2'
         ```
 
    - Gradle/Kotlin
 
         ```kotlin
-        implementation("io.milvus:milvus-sdk-java:2.5.1")
+        implementation("io.milvus:milvus-sdk-java:2.5.2")
         ```
-        
+
+From v2.5.2, milvus Java SDK is split into two packages: milvus-sdk-java and milvus-sdk-java-bulkwriter, because BulkWriter requires quite a lot of dependencies. If you don't need BulkWriter tool, you can ignore the milvus-sdk-java-bulkwriter package.
+To use BulkWriter, import milvus-sdk-java-bulkwriter to your project.
+
+- Apache Maven
+
+    ```xml
+     <dependency>
+         <groupId>io.milvus</groupId>
+         <artifactId>milvus-sdk-java-bulkwriter</artifactId>
+         <version>2.5.2</version>
+     </dependency>
+    ```
+
+- Gradle/Groovy
+
+     ```groovy
+     implementation 'io.milvus:milvus-sdk-java-bulkwriter:2.5.2'
+     ```
+
+- Gradle/Kotlin
+
+     ```kotlin
+     implementation("io.milvus:milvus-sdk-java-bulkwriter:2.5.2")
+     ```     
+
 ### Examples
 
 Please refer to [examples](https://github.com/milvus-io/milvus-sdk-java/tree/master/examples) folder for Java SDK examples.

examples/main/java/io/milvus/v1/ClientPoolExample.java

@@ -106,11 +106,19 @@ public static void createCollection(MilvusClientV1Pool pool) {
     public static Thread runInsertThread(MilvusClientV1Pool pool, String clientName, int repeatRequests) {
         Thread t = new Thread(() -> {
             Gson gson = new Gson();
-            Random rand = new Random();
             for (int i = 0; i < repeatRequests; i++) {
-                MilvusClient client = pool.getClient(clientName);
+                MilvusClient client = null;
+                while(client == null) {
+                    try {
+                        // getClient() might exceeds the borrowMaxWaitMillis and throw exception
+                        // retry to call until it return a client
+                        client = pool.getClient(clientName);
+                    } catch (Exception e) {
+                        System.out.printf("Failed to get client, will retry, error: %s%n", e.getMessage());
+                    }
+                }
                 try {
-                    int rowCount = rand.nextInt(10) + 10;
+                    int rowCount = 1;
                     List<JsonObject> rows = new ArrayList<>();
                     for (int j = 0; j < rowCount; j++) {
                         JsonObject row = new JsonObject();
@@ -141,7 +149,16 @@ public static Thread runInsertThread(MilvusClientV1Pool pool, String clientName,
     public static Thread runSearchThread(MilvusClientV1Pool pool, String clientName, int repeatRequests) {
         Thread t = new Thread(() -> {
             for (int i = 0; i < repeatRequests; i++) {
-                MilvusClient client = pool.getClient(clientName);
+                MilvusClient client = null;
+                while(client == null) {
+                    try {
+                        // getClient() might exceeds the borrowMaxWaitMillis and throw exception
+                        // retry to call until it return a client
+                        client = pool.getClient(clientName);
+                    } catch (Exception e) {
+                        System.out.printf("Failed to get client, will retry, error: %s%n", e.getMessage());
+                    }
+                }
                 try {
                     R<SearchResults> searchRet = client.search(SearchParam.newBuilder()
                             .withCollectionName(CollectionName)
@@ -191,7 +208,7 @@ public static void main(String[] args) throws InterruptedException {
         createCollection(pool);
 
         List<Thread> threadList = new ArrayList<>();
-        int threadCount = 10;
+        int threadCount = 100;
         int repeatRequests = 100;
         long start = System.currentTimeMillis();
         for (int k = 0; k < threadCount; k++) {
@@ -212,7 +229,7 @@ public static void main(String[] args) throws InterruptedException {
         }
         long end = System.currentTimeMillis();
         System.out.printf("%d insert requests and %d search requests finished in %.3f seconds%n",
-                threadCount*repeatRequests, threadCount*repeatRequests, (end-start)*0.001);
+                threadCount*repeatRequests*3, threadCount*repeatRequests*3, (end-start)*0.001);
         System.out.printf("Total %d idle clients and %d active clients%n",
                 pool.getTotalIdleClientNumber(), pool.getTotalActiveClientNumber());
         pool.clear(); // clear idle clients

examples/main/java/io/milvus/v2/ClientPoolExample.java

@@ -40,7 +40,6 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Random;
 
 public class ClientPoolExample {
     public static String CollectionName = "java_sdk_example_pool_v2";
@@ -79,11 +78,19 @@ public static void createCollection(MilvusClientV2Pool pool) {
     public static Thread runInsertThread(MilvusClientV2Pool pool, String clientName, int repeatRequests) {
         Thread t = new Thread(() -> {
             Gson gson = new Gson();
-            Random rand = new Random();
             for (int i = 0; i < repeatRequests; i++) {
-                MilvusClientV2 client = pool.getClient(clientName);
+                MilvusClientV2 client = null;
+                while(client == null) {
+                    try {
+                        // getClient() might exceeds the borrowMaxWaitMillis and throw exception
+                        // retry to call until it return a client
+                        client = pool.getClient(clientName);
+                    } catch (Exception e) {
+                        System.out.printf("Failed to get client, will retry, error: %s%n", e.getMessage());
+                    }
+                }
                 try {
-                    int rowCount = rand.nextInt(10) + 10;
+                    int rowCount = 1;
                     List<JsonObject> rows = new ArrayList<>();
                     for (int j = 0; j < rowCount; j++) {
                         JsonObject row = new JsonObject();
@@ -110,7 +117,16 @@ public static Thread runInsertThread(MilvusClientV2Pool pool, String clientName,
     public static Thread runSearchThread(MilvusClientV2Pool pool, String clientName, int repeatRequests) {
         Thread t = new Thread(() -> {
             for (int i = 0; i < repeatRequests; i++) {
-                MilvusClientV2 client = pool.getClient(clientName);
+                MilvusClientV2 client = null;
+                while(client == null) {
+                    try {
+                        // getClient() might exceeds the borrowMaxWaitMillis and throw exception
+                        // retry to call until it return a client
+                        client = pool.getClient(clientName);
+                    } catch (Exception e) {
+                        System.out.printf("Failed to get client, will retry, error: %s%n", e.getMessage());
+                    }
+                }
                 try {
                     SearchResp result = client.search(SearchReq.builder()
                             .collectionName(CollectionName)
@@ -154,7 +170,7 @@ public static void main(String[] args) throws InterruptedException {
         createCollection(pool);
 
         List<Thread> threadList = new ArrayList<>();
-        int threadCount = 10;
+        int threadCount = 100;
         int repeatRequests = 100;
         long start = System.currentTimeMillis();
         for (int k = 0; k < threadCount; k++) {
@@ -175,7 +191,7 @@ public static void main(String[] args) throws InterruptedException {
         }
         long end = System.currentTimeMillis();
         System.out.printf("%d insert requests and %d search requests finished in %.3f seconds%n",
-                threadCount*repeatRequests, threadCount*repeatRequests, (end-start)*0.001);
+                threadCount*repeatRequests*3, threadCount*repeatRequests*3, (end-start)*0.001);
         System.out.printf("Total %d idle clients and %d active clients%n",
                 pool.getTotalIdleClientNumber(), pool.getTotalActiveClientNumber());
 

examples/pom.xml

@@ -25,7 +25,7 @@
 
     <groupId>io.milvus</groupId>
     <artifactId>milvus-sdk-java-examples</artifactId>
-    <version>2.5.1</version>
+    <version>2.5.2</version>
 
     <build>
         <plugins>
@@ -64,7 +64,12 @@
         <dependency>
             <groupId>io.milvus</groupId>
             <artifactId>milvus-sdk-java</artifactId>
-            <version>2.5.1</version>
+            <version>2.5.2</version>
+        </dependency>
+        <dependency>
+            <groupId>io.milvus</groupId>
+            <artifactId>milvus-sdk-java-bulkwriter</artifactId>
+            <version>2.5.2</version>
         </dependency>
         <dependency>
             <groupId>org.tensorflow</groupId>