diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000000..a2e46878cf
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,2 @@
+java/fury-benchmark export-ignore
+integration_tests/perftests export-ignore
\ No newline at end of file
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 83f0693ac7..fcbfc0867f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -120,6 +120,10 @@ jobs:
         with:
           java-version: 8
           distribution: 'temurin'
+      - name: Install fury-benchmark jar
+        run: |
+          cd java && mvn -T10 -B --no-transfer-progress clean install -DskipTests && cd -
+          cd java/fury-benchmark && mvn -T10 -B --no-transfer-progress clean install -DskipTests -Pjmh && cd -
       - name: Run CI
         run: ./ci/run_ci.sh integration_tests
   javascript:
diff --git a/.gitignore b/.gitignore
index 501ec335d8..5bc008505c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,10 +15,10 @@ cython_debug
 testresult
 java/target
 benchmark/target
-fury-benchmark/target
-**/target/**/io/ray/**
+**/target/**/**
 java/**/.code/
 java/**/dependency-reduced-pom.xml
+**/pom.xml.versionsBackup
 **/.code/
 **/*.iml
 **/*.html
diff --git a/ci/deploy.sh b/ci/deploy.sh
index 9959d77c61..bb9d4be80f 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -84,6 +84,8 @@ bump_java_version() {
   cd "$ROOT/java"
   echo "Set fury java version to $version"
   mvn versions:set -DnewVersion="$version"
+  cd "$ROOT/java/fury-benchmark"
+  mvn versions:set -DnewVersion="$version"
   echo "Set fury integration_tests version to $version"
   cd "$ROOT/integration_tests/jdk_compatibility_tests"
   mvn versions:set -DnewVersion="$version"
diff --git a/ci/format.sh b/ci/format.sh
index 5ae5f9ed29..433dc47b96 100755
--- a/ci/format.sh
+++ b/ci/format.sh
@@ -185,6 +185,8 @@ format_java() {
       cd "$ROOT/java"
       mvn -T10 --no-transfer-progress spotless:apply
       mvn -T10 --no-transfer-progress checkstyle:check
+      cd "$ROOT/java/fury-benchmark"
+      mvn -T10 --no-transfer-progress spotless:apply
       cd "$ROOT/integration_tests"
       dirs=("graalvm_tests" "jdk_compatibility_tests" "latest_jdk_tests" "perftests")
       for d in "${dirs[@]}" ; do
diff --git a/ci/run_ci.sh b/ci/run_ci.sh
index 260d0d05fb..1a7173ac74 100755
--- a/ci/run_ci.sh
+++ b/ci/run_ci.sh
@@ -101,8 +101,6 @@ graalvm_test() {
 integration_tests() {
   cd "$ROOT"/java
   mvn -T10 -B --no-transfer-progress clean install -DskipTests
-  echo "Start perf tests"
-  cd "$ROOT"/integration_tests/perftests
   echo "Start latest jdk tests"
   cd "$ROOT"/integration_tests/latest_jdk_tests
   echo "latest_jdk_tests: JDK 21"
@@ -131,7 +129,7 @@ jdk17_plus_tests() {
   echo "Executing fury java tests"
   cd "$ROOT/java"
   set +e
-  mvn -T10 --batch-mode --no-transfer-progress test install -pl '!fury-format,!fury-testsuite,!fury-benchmark'
+  mvn -T10 --batch-mode --no-transfer-progress test install -pl '!fury-format,!fury-testsuite'
   testcode=$?
   if [[ $testcode -ne 0 ]]; then
     exit $testcode
diff --git a/integration_tests/perftests/README.md b/integration_tests/perftests/README.md
index 7bac51d279..22e3ff87b7 100644
--- a/integration_tests/perftests/README.md
+++ b/integration_tests/perftests/README.md
@@ -10,9 +10,15 @@ proto code can be generated by maven plugin.
 Flatbuffers generated is short, just add generated files to repo directly.
 
 # Run tests
+This benchmark use [jmh](https://github.com/openjdk/jmh) as benchmark tool. [jmh](https://github.com/openjdk/jmh) is
+licensed under GPL V2 with CLASSPATH exception, the usage can't be included in apache source/binary release unless
+as an optional feature. So Fury make it as an optional dependency and you must enable `jmh` profile to activate it.
+
 ```bash
 cd java && mvn -T10 install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dmaven.javadoc.skip && cd ../integration_tests/perftests
-mvn -T10 compile
+# use `-Pjmh` to download jmh dependencies, we mark it as optional
+# since jmh is licenced under GPL V2 andn not comply with the license policy of ASF.
+mvn -T10 compile -Pjmh
 #mvn exec:java -Dexec.mainClass="org.apache.fury.integration_tests.UserTypeSerializeSuite" -Dexec.args="-f 1 -wi 0 -i 1 -t 1 -w 1s -r 1s -rf csv"
-mvn exec:java -Dexec.mainClass="org.apache.fury.integration_tests.UserTypeSerializeSuite"
+mvn exec:java -Dexec.mainClass="org.apache.fury.integration_tests.UserTypeSerializeSuite" -Pjmh
 ```
\ No newline at end of file
diff --git a/integration_tests/perftests/pom.xml b/integration_tests/perftests/pom.xml
index 798d7b5f5c..9a958e821c 100644
--- a/integration_tests/perftests/pom.xml
+++ b/integration_tests/perftests/pom.xml
@@ -55,17 +55,6 @@
   </repositories>
 
   <dependencies>
-    <dependency>
-      <groupId>org.apache.fury</groupId>
-      <artifactId>fury-benchmark</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.openjdk.jmh</groupId>
-      <artifactId>jmh-generator-annprocess</artifactId>
-      <version>${jmh.version}</version>
-      <scope>provided</scope>
-    </dependency>
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
@@ -78,6 +67,25 @@
     </dependency>
   </dependencies>
 
+  <profiles>
+    <profile>
+      <id>jmh</id>
+      <dependencies>
+        <dependency>
+          <groupId>org.openjdk.jmh</groupId>
+          <artifactId>jmh-core</artifactId>
+          <version>${jmh.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.openjdk.jmh</groupId>
+          <artifactId>jmh-generator-annprocess</artifactId>
+          <version>${jmh.version}</version>
+          <scope>provided</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
+
   <build>
     <plugins>
       <plugin>
diff --git a/java/fury-benchmark/README.md b/java/fury-benchmark/README.md
index 4ed3691d2f..bcedc2b631 100644
--- a/java/fury-benchmark/README.md
+++ b/java/fury-benchmark/README.md
@@ -13,14 +13,18 @@ Fury Java Benchmark contains benchmarks for:
 > Kryo benchmark suite is based on [Kryo benchmark](https://github.com/EsotericSoftware/kryo/tree/master/benchmarks).
 
 # How to run
+This benchmark use [jmh](https://github.com/openjdk/jmh) as benchmark tool. [jmh](https://github.com/openjdk/jmh) is 
+licensed under GPL V2 with CLASSPATH exception, the usage can't be included in apache source/binary release unless
+as an optional feature. So Fury make it as an optional dependency and you must enable `jmh` profile to activate it.
 
 ```bash
 # Install fury
 cd ../java && mvn install -DskipTests && cd -
 
 # build benchmark jar
-mvn package
-
+# use `-Pjmh` to download jmh dependencies, we mark it as optional
+# since jmh is licenced under GPL V2 andn not comply with the license policy of ASF.
+mvn package -Pjmh
 # run benchmark
 nohup java -jar target/benchmarks.jar -f 5 -wi 3 -i 5 -t 1 -w 3s -r 5s -rf csv >bench.log 2>&1 &
 java -jar target/benchmarks.jar "io.*\.deserialize$" -f 1 -wi 1 -i 3 -t 1 -w 2s -r 2s -rf csv
diff --git a/java/fury-benchmark/pom.xml b/java/fury-benchmark/pom.xml
index 89ac0d4034..a532727785 100644
--- a/java/fury-benchmark/pom.xml
+++ b/java/fury-benchmark/pom.xml
@@ -50,18 +50,6 @@
   </properties>
 
   <dependencies>
-    <dependency>
-      <groupId>org.openjdk.jmh</groupId>
-      <artifactId>jmh-core</artifactId>
-      <version>${jmh.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.openjdk.jmh</groupId>
-      <artifactId>jmh-generator-annprocess</artifactId>
-      <version>${jmh.version}</version>
-      <scope>provided</scope>
-    </dependency>
-
     <dependency>
       <groupId>org.apache.fury</groupId>
       <artifactId>fury-core</artifactId>
@@ -153,6 +141,25 @@
     </dependency>
   </dependencies>
 
+  <profiles>
+    <profile>
+      <id>jmh</id>
+      <dependencies>
+        <dependency>
+          <groupId>org.openjdk.jmh</groupId>
+          <artifactId>jmh-core</artifactId>
+          <version>${jmh.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.openjdk.jmh</groupId>
+          <artifactId>jmh-generator-annprocess</artifactId>
+          <version>${jmh.version}</version>
+          <scope>provided</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
+
   <build>
     <plugins>
       <plugin>
diff --git a/java/pom.xml b/java/pom.xml
index 03339ab9c2..ba83466732 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -61,7 +61,6 @@
     <module>fury-core</module>
     <module>fury-test-core</module>
     <module>fury-testsuite</module>
-    <module>fury-benchmark</module>
   </modules>
 
   <properties>