feat(java): Put java and spark libraries within a directory maven-projects to help easily manage and extend the multi-module maven projects

.github/workflows/java.yml

@@ -24,13 +24,13 @@ on:
     branches:
       - main
     paths:
-      - 'java/**'
+      - 'maven-projects/java/**'
       - '.github/workflows/java.yml'
   pull_request:
     branches:
       - main
     paths:
-      - 'java/**'
+      - 'maven-projects/java/**'
       - '.github/workflows/java.yml'
 
 concurrency:
@@ -61,22 +61,22 @@ jobs:
           sudo apt-get install llvm-11 clang-11 lld-11 libclang-11-dev libz-dev -y
 
       - name: Code Format Check
-        working-directory: java
+        working-directory: maven-projects/java
         run: |
           export JAVA_HOME=${JAVA_HOME_11_X64}
           mvn --no-transfer-progress spotless:check
 
       - name: Build Java Docs
-        working-directory: java
+        working-directory: maven-projects/java
         run: |
           export JAVA_HOME=${JAVA_HOME_11_X64}
           mvn --no-transfer-progress javadoc:javadoc
 
       - name: Run test
-        working-directory: java
+        working-directory: maven-projects/java
         run: |
           # Temporarily using Java 8, related issue: https://github.com/apache/incubator-graphar/issues/277
           export JAVA_HOME=${JAVA_HOME_8_X64}
           export LLVM11_HOME=/usr/lib/llvm-11
-          export GAR_TEST_DATA=$PWD/../testing/
+          export GAR_TEST_DATA=$PWD/../../testing/
           mvn --no-transfer-progress clean test -Dspotless.check.skip=true

.github/workflows/pyspark.yml

@@ -24,14 +24,14 @@ on:
     branches:
       - main
     paths:
-      - 'spark/**'
+      - 'maven-projects/spark/**'
       - 'pyspark/**'
       - '.github/workflows/pyspark.yml'
   pull_request:
     branches:
       - main
     paths:
-      - 'spark/**'
+      - 'maven-projects/spark/**'
       - 'pyspark/**'
       - '.github/workflows/pyspark.yml'
 

.github/workflows/spark.yaml

@@ -24,13 +24,13 @@ on:
     branches:
       - main
     paths:
-      - 'spark/**'
+      - 'maven-projects/spark/**'
       - '.github/workflows/spark.yaml'
   pull_request:
     branches:
       - main
     paths:
-      - 'spark/**'
+      - 'maven-projects/spark/**'
       - '.github/workflows/spark.yaml'
 
 concurrency:
@@ -57,20 +57,20 @@ jobs:
           submodules: true
 
     - name: Code Format Check
-      working-directory: spark
+      working-directory: maven-projects/spark
       run: |
         export JAVA_HOME=${JAVA_HOME_11_X64}
         mvn --no-transfer-progress spotless:check
 
     - name: Build GraphAr Spark
-      working-directory: spark
+      working-directory: maven-projects/spark
       run: |
         export JAVA_HOME=${JAVA_HOME_11_X64}
         echo "Build ${{ matrix.mvn-profile }}"
         mvn --no-transfer-progress clean package -DskipTests -Dspotless.check.skip=true -P ${{ matrix.mvn-profile }}
 
     - name: Build Spark Docs
-      working-directory: spark
+      working-directory: maven-projects/spark
       run: |
         export JAVA_HOME=${JAVA_HOME_11_X64}
         echo "Build ${{ matrix.mvn-profile }}"
@@ -79,15 +79,15 @@ jobs:
         mvn --no-transfer-progress scala:doc
 
     - name: Run test
-      working-directory: spark
+      working-directory: maven-projects/spark
       run: |
         export JAVA_HOME=${JAVA_HOME_11_X64}
         export SPARK_TESTING=1
         echo "Test ${{ matrix.mvn-profile }}"
         mvn test --no-transfer-progress -Dspotless.check.skip=true -P ${{ matrix.mvn-profile }}
 
     - name: Run Neo4j2GraphAr example
-      working-directory: spark
+      working-directory: maven-projects/spark
       run: |
         export JAVA_HOME=${JAVA_HOME_11_X64}
         scripts/get-spark-to-home.sh ${{ matrix.spark }} ${{ matrix.spark-hadoop }}
@@ -112,7 +112,7 @@ jobs:
         scripts/run-graphar2neo4j.sh
 
     - name: Run Nebula2GraphAr example
-      working-directory: spark
+      working-directory: maven-projects/spark
       run: |
         export JAVA_HOME=${JAVA_HOME_11_X64}
         scripts/get-nebula-to-home.sh
@@ -138,7 +138,7 @@ jobs:
         scripts/run-graphar2nebula.sh
 
     - name: Run Neo4j importer
-      working-directory: spark
+      working-directory: maven-projects/spark
       run: |
         export JAVA_HOME=${JAVA_HOME_11_X64}
         export SPARK_HOME="${HOME}/${{ matrix.spark-hadoop }}"

README.md

@@ -204,7 +204,7 @@ programming language.
 ### The C++ Library
 
 See [GraphAr C++
-Library](https://github.com/apache/incubator-graphar/tree/main/cpp) for
+Library](./cpp) for
 details about the building of the C++ library.
 
 ### The Java Library
@@ -213,20 +213,20 @@ The GraphAr Java library is created with bindings to the C++ library
 (currently at version v0.10.0), utilizing
 [Alibaba-FastFFI](https://github.com/alibaba/fastFFI) for
 implementation. See [GraphAr Java
-Library](https://github.com/apache/incubator-graphar/tree/main/java) for
+Library](./maven-projects/java) for
 details about the building of the Java library.
 
 ### The Spark Library
 
 See [GraphAr Spark
-Library](https://github.com/apache/incubator-graphar/tree/main/spark)
+Library](./maven-projects/spark)
 for details about the Spark library.
 
 ### The PySpark Library
 
 The GraphAr PySpark library is developed as bindings to the GraphAr
 Spark library. See [GraphAr PySpark
-Library](https://github.com/apache/incubator-graphar/tree/main/pyspark)
+Library](./pyspark)
 for details about the PySpark library.
 
 ## Contributing

docs/libraries/java/how_to_develop_java.md

@@ -80,7 +80,7 @@ Please refer to
 ## How To Test
 
 ```bash
-$ export GAR_TEST_DATA=$PWD/../testing/
+$ export GAR_TEST_DATA=$PWD/../../testing/
 $ mvn clean test
 ```
 

docs/libraries/java/java.md

@@ -69,7 +69,7 @@ directory:
 $ git clone https://github.com/apache/incubator-graphar.git
 $ cd incubator-graphar
 $ git submodule update --init
-$ cd java
+$ cd maven-projects/java
 ```
 
 Compile package:
@@ -120,7 +120,7 @@ if (!graphInfoResult.hasError()) {
 ```
 
 See [test for
-graphinfo](https://github.com/apache/incubator-graphar/tree/main/java/src/test/java/org/apache/graphar/graphinfo)
+graphinfo](https://github.com/apache/incubator-graphar/blob/main/maven-projects/java/src/test/java/org/apache/graphar/graphinfo)
 for the complete example.
 
 ### Writers
@@ -177,7 +177,7 @@ writer.sortAndWriteAdjListTable(table, 0, 0); // Write adj list of vertex chunk
 ```
 
 See [test for
-writers](https://github.com/apache/incubator-graphar/tree/main/java/src/test/java/org/apache/graphar/writers)
+writers](https://github.com/apache/incubator-graphar/blob/main/maven-projects/java/src/test/java/org/apache/graphar/writers)
 for the complete example.
 
 ### Readers
@@ -211,5 +211,5 @@ StdPair<Long, Long> range = reader.getRange().value();
 ```
 
 See [test for
-readers](https://github.com/apache/incubator-graphar/tree/main/java/src/test/java/org/apache/graphar/readers)
+readers](https://github.com/apache/incubator-graphar/blob/main/maven-projects/java/src/test/java/org/apache/graphar/readers)
 for the complete example.

docs/libraries/spark/examples.md

@@ -206,8 +206,8 @@ See [GraphAr2Neo4j.scala][graphar2neo4j] for the complete example.
 
 :::
 
-[test-graph-transformer]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/test/scala/org/apache/graphar/TestGraphTransformer.scala
-[transformer-example]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/test/scala/org/apache/graphar/TransformExample.scala
-[compute-example]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/test/scala/org/apache/graphar/ComputeExample.scala
-[neo4j2graphar]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/main/scala/org/apache/graphar/example/Neo4j2GraphAr.scala
-[graphar2neo4j]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/main/scala/org/apache/graphar/example/GraphAr2Neo4j.scala
+[test-graph-transformer]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/test/scala/org/apache/graphar/TestGraphTransformer.scala
+[transformer-example]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/test/scala/org/apache/graphar/TransformExample.scala
+[compute-example]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/test/scala/org/apache/graphar/ComputeExample.scala
+[neo4j2graphar]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/main/scala/org/apache/graphar/example/Neo4j2GraphAr.scala
+[graphar2neo4j]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/main/scala/org/apache/graphar/example/GraphAr2Neo4j.scala

docs/libraries/spark/spark.md

@@ -226,12 +226,12 @@ For more information on usage, please refer to the examples:
 - [Neo4j2GraphAr.scala][neo4j2graphar] and [GraphAr2Neo4j.scala][graphar2neo4j] are examples to conduct data importing/exporting for Neo4j.
 
 
-[test-graph-info]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/test/scala/org/apache/graphar/TestGraphInfo.scala
-[test-index-generator]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/test/scala/org/apache/graphar/TestIndexGenerator.scala
-[test-writer]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/test/scala/org/apache/graphar/TestWriter.scala
-[test-reader]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/test/scala/org/apache/graphar/TestReader.scala
-[test-graph-transformer]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/test/scala/org/apache/graphar/TestGraphTransformer.scala
-[compute-example]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/test/scala/org/apache/graphar/ComputeExample.scala
-[transform-example]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/test/scala/org/apache/graphar/TransformExample.scala
-[neo4j2graphar]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/main/scala/org/apache/graphar/example/Neo4j2GraphAr.scala
-[graphar2neo4j]: https://github.com/apache/incubator-graphar/blob/main/spark/graphar/src/main/scala/org/apache/graphar/example/GraphAr2Neo4j.scala
+[test-graph-info]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/test/scala/org/apache/graphar/TestGraphInfo.scala
+[test-index-generator]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/test/scala/org/apache/graphar/TestIndexGenerator.scala
+[test-writer]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/test/scala/org/apache/graphar/TestWriter.scala
+[test-reader]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/test/scala/org/apache/graphar/TestReader.scala
+[test-graph-transformer]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/test/scala/org/apache/graphar/TestGraphTransformer.scala
+[compute-example]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/test/scala/org/apache/graphar/ComputeExample.scala
+[transform-example]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/test/scala/org/apache/graphar/TransformExample.scala
+[neo4j2graphar]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/main/scala/org/apache/graphar/example/Neo4j2GraphAr.scala
+[graphar2neo4j]: https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/main/scala/org/apache/graphar/example/GraphAr2Neo4j.scala

java/README.md → maven-projects/java/README.md

@@ -46,7 +46,7 @@ Make the graphar-java-library directory as the current working directory:
   $ git clone https://github.com/apache/incubator-graphar.git
   $ cd incubator-graphar
   $ git submodule update --init
-  $ cd java
+  $ cd maven-projects/java
 ```
 
 Compile package:
@@ -82,4 +82,4 @@ The API document will be generated in the `target/site/apidocs` directory.
 
 ## How to use
 
-Please refer to [GraphAr Java Library Documentation](https://graphar.apache.org/GraphAr/user-guide/java-lib.html).
+Please refer to [GraphAr Java Library Documentation](https://graphar.apache.org/docs/libraries/java/).

pom.xml → maven-projects/pom.xml

@@ -30,7 +30,7 @@
 
   <name>Apache GraphAr Root POM</name>
   <modules>
-    <module>java</module>
+    <module>info</module>
     <module>spark</module>
   </modules>
 

spark/README.md → maven-projects/spark/README.md

@@ -22,24 +22,24 @@ repository and navigated to the ``spark`` subdirectory:
     $ git clone https://github.com/apache/incubator-graphar.git
     $ cd incubator-graphar
     $ git submodule update --init
-    $ cd spark
+    $ cd mavens-projects/spark
 ```
 
 
 Build the package:
 
 ```bash
-    $ mvn clean package -DskipTests
+    $ mvn clean install -DskipTests
 ```
 
-GraphAr Spark uses Maven Profiles to support multiple Spark Versions. By default it is built with Spark 3.2.x or profile `datasources-32`. To built with Spark 3.3.4 use `-P datasources-33` (`mvn clean package -DskipTests -P datasources-33`).
+GraphAr Spark uses Maven Profiles to support multiple Spark Versions. By default it is built with Spark 3.2.x or profile `datasources-32`. To built with Spark 3.3.4 use `-P datasources-33` (`mvn clean install -DskipTests -P datasources-33`).
 
 After compilation, the package file graphar-x.x.x-SNAPSHOT-shaded.jar is generated in the directory ``spark/graphar/target/``.
 
 Build the package and run the unit tests:
 
 ```bash
-    $ mvn clean package
+    $ mvn clean install
 ```
 
 Build and run the unit tests:
@@ -231,4 +231,4 @@ The example will import the basketballplayer graph from GraphAr to NebulaGraph a
 
 ## How to use
 
-Please refer to our [GraphAr Spark Library Documentation](https://graphar.apache.org/GraphAr/spark/spark-lib.html).
+Please refer to our [GraphAr Spark Library Documentation](https://graphar.apache.org/docs/libraries/spark/).

maven-projects/spark/graphar/src/test/resources/gar-test

@@ -0,0 +1 @@
+../../../../../../testing

pyspark/Makefile

@@ -18,7 +18,7 @@
 .PHONY: install_test
 install_test:
 		export JAVA_HOME=${JAVA_HOME_11_X64}
-		cd ../spark && mvn --no-transfer-progress clean package -DskipTests -Dspotless.check.skip=true && cd ../pyspark
+		cd ../maven-projects/spark && mvn --no-transfer-progress clean package -DskipTests -Dspotless.check.skip=true && cd ../../pyspark 
 		export PYSPARK_HADOOP_VERSION=3.2
 		poetry install --with=spark,tests
 

pyspark/tests/conftest.py

@@ -20,7 +20,7 @@
 import pytest
 from pyspark.sql import SparkSession
 
-JARS_PATH = Path(__file__).parent.parent.parent.joinpath("spark").joinpath("graphar").joinpath("target")
+JARS_PATH = Path(__file__).parent.parent.parent.joinpath("maven-projects/spark").joinpath("graphar").joinpath("target")
 GRAPHAR_SHADED_JAR_PATH = None
 
 for jar_file in JARS_PATH.glob("*.jar"):