Add spark tool api reference to doc (#59)

.github/workflows/docs.yml

@@ -47,6 +47,18 @@ jobs:
           make doc
           popd
 
+      - name: preview
+        uses: afc163/surge-preview@v1
+        id: preview_step
+        with:
+          surge_token: ${{ secrets.SURGE_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          dist: docs/_build/html
+          build: echo "done"
+
+      - name: Get the preview_url
+        run: echo "url => ${{ steps.preview_step.outputs.preview_url }}"
+
       - name: Commit Doc
         if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphAr' }}
         run: |

CMakeLists.txt

@@ -194,7 +194,7 @@ build_gar()
 
 # ------------------------------------------------------------------------------
 # build example
-# ------------------------------------------------------------------------------ 
+# ------------------------------------------------------------------------------
 if (BUILD_EXAMPLES)
     find_package(Boost REQUIRED COMPONENTS graph)
 
@@ -305,6 +305,8 @@ if(doxygen_EXECUTABLE AND sphinx_build_EXECUTABLE)
         COMMAND ${CMAKE_COMMAND} -E make_directory _build
         COMMAND ${doxygen_EXECUTABLE}
         COMMAND ${sphinx_build_EXECUTABLE} . _build/html
+        COMMAND cd ../spark && mvn scala:doc && rm -fr ${PROJECT_SOURCE_DIR}/docs/_build/html/reference/spark-api &&
+         cp -fr target/site/scaladocs ${PROJECT_SOURCE_DIR}/docs/_build/html/reference/spark-api
         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/docs
         VERBATIM
     )

docs/index.rst

@@ -34,6 +34,6 @@
 .. toctree::
    :maxdepth: 1
    :caption: API Reference
-   :hidden:
 
-   api-reference.rst
+   reference/api-reference-cpp.rst
+   Spark API Reference <reference/spark-api/index>

docs/api-reference.rst → docs/reference/api-reference-cpp.rst

@@ -1,5 +1,5 @@
-API Reference
-==============
+C++ API Reference
+==================
 
 .. _cpp-api:
 

docs/reference/spark-api/index.rst

@@ -0,0 +1,4 @@
+Spark API Reference (scaladoc)
+==============================
+
+Stub page for the Spark reference docs; actual source is located in the spark-api/ directory.

include/gar/graph_info.h

@@ -504,7 +504,7 @@ class EdgeInfo {
   }
 
   /**
-   * @brief Return property group that contains certain property and with adj
+   * @brief Return property group that contains property and with adj
    * list type
    *
    * @param property property name

include/gar/reader/arrow_chunk_reader.h

@@ -141,7 +141,7 @@ class AdjListArrowChunkReader {
    * @brief Initialize the AdjListArrowChunkReader.
    *
    * @param edge_info The edge info that describes the edge type.
-   * @param adj_list_type The adj list type for the edges.
+   * @param adj_list_type The adj list type for the edge.
    * @param prefix The absolute prefix.
    * @param vertex_chunk_index The vertex chunk index, default is 0.
    */

include/gar/utils/data_type.h

@@ -32,13 +32,13 @@ namespace GAR_NAMESPACE_INTERNAL {
 
 /// \brief Main data type enumeration
 enum class Type {
-  /// Boolean as 1 bit, LSB bit-packed ordering
+  /// Boolean
   BOOL = 0,
 
-  /// Signed 32-bit little-endian integer
+  /// Signed 32-bit integer
   INT32,
 
-  /// Signed 64-bit little-endian integer
+  /// Signed 64-bit integer
   INT64,
 
   /// 4-byte floating point value
@@ -47,7 +47,7 @@ enum class Type {
   /// 8-byte floating point value
   DOUBLE,
 
-  /// UTF8 variable-length string as List<Char>
+  /// UTF8 variable-length string
   STRING,
 
   /// User-defined data type

spark/pom.xml

@@ -189,6 +189,17 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <version>4.8.0</version>
+                <configuration>
+                    <jvmArgs>
+                        <jvmArg>-Xms64m</jvmArg>
+                        <jvmArg>-Xmx1024m</jvmArg>
+                    </jvmArgs>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
     <packaging>jar</packaging>

spark/src/main/java/com/alibaba/graphar/GeneralParams.java

@@ -15,6 +15,7 @@
 
 package com.alibaba.graphar;
 
+/** General constant parameters for graphar. */
 public class GeneralParams {
     // column name
     public static final String vertexIndexCol = "_graphArVertexIndex";

spark/src/main/scala/com/alibaba/graphar/EdgeInfo.scala

@@ -20,6 +20,7 @@ import org.yaml.snakeyaml.Yaml
 import org.yaml.snakeyaml.constructor.Constructor
 import scala.beans.BeanProperty
 
+/** Edge info is a class to store the edge meta information. */
 class EdgeInfo() {
   @BeanProperty var src_label: String = ""
   @BeanProperty var edge_label: String = ""
@@ -32,6 +33,11 @@ class EdgeInfo() {
   @BeanProperty var adj_lists = new java.util.ArrayList[AdjList]()
   @BeanProperty var version: String = ""
 
+  /** Check if the edge info supports the adj list type.
+   *
+   * @param adj_list_type adjList type in gar to check.
+   * @return true if edge info supports the adj list type, otherwise return false.
+   */
   def containAdjList(adj_list_type: AdjListType.Value): Boolean = {
     val tot: Int = adj_lists.size
     for ( k <- 0 to tot - 1 ) {
@@ -42,6 +48,12 @@ class EdgeInfo() {
     return false
   }
 
+  /** Get path prefix of adj list type.
+   *
+   * @param adj_list_type The input adj list type in gar.
+   * @return path prefix of the adj list type, if edge info not support the adj list type,
+   *         raise an IllegalArgumentException error.
+   */
   def getAdjListPrefix(adj_list_type: AdjListType.Value): String = {
     val tot: Int = adj_lists.size
     for ( k <- 0 to tot - 1 ) {
@@ -57,6 +69,12 @@ class EdgeInfo() {
     throw new IllegalArgumentException
   }
 
+  /** Get the adj list topology chunk file type of adj list type.
+   *
+   * @param adj_list_type the input adj list type.
+   * @return file format type in gar of the adj list type, if edge info not support the adj list type,
+   *         raise an IllegalArgumentException error.
+   */
   def getAdjListFileType(adj_list_type: AdjListType.Value): FileType.Value = {
     val tot: Int = adj_lists.size
     for ( k <- 0 to tot - 1 ) {
@@ -68,6 +86,12 @@ class EdgeInfo() {
     throw new IllegalArgumentException
   }
 
+  /** Get the property groups of adj list type.
+   *
+   * @param adj_list_type the input adj list type.
+   * @return property group of the input adj list type, if edge info not support the adj list type,
+   *         raise an IllegalArgumentException error.
+   */
   def getPropertyGroups(adj_list_type: AdjListType.Value): java.util.ArrayList[PropertyGroup] = {
     val tot: Int = adj_lists.size
     for ( k <- 0 to tot - 1 ) {
@@ -79,6 +103,14 @@ class EdgeInfo() {
     throw new IllegalArgumentException
   }
 
+  /** Check if the edge info contains the property group in cerain adj list structure.
+   *
+   * @param property_group the property group to check.
+   * @param adj_list_type the type of adj list structure.
+   * @return true if the edge info contains the property group in cerain adj list structure.
+   *         If edge info not support the given adj list type or not contains the proerpty group in the adj list structure,
+   *         return false.
+   */
   def containPropertyGroup(property_group: PropertyGroup, adj_list_type: AdjListType.Value): Boolean = {
     val tot: Int = adj_lists.size
     for ( k <- 0 to tot - 1 ) {
@@ -97,6 +129,11 @@ class EdgeInfo() {
     return false
   }
 
+  /**  Check if the edge info contains the property.
+   *
+   * @param property_name name of the property.
+   * @return true if edge info contains the property, otherwise false.
+   */
   def containProperty(property_name: String): Boolean = {
     val tot: Int = adj_lists.size
     for ( k <- 0 to tot - 1 ) {
@@ -117,6 +154,13 @@ class EdgeInfo() {
     return false
   }
 
+  /** Get property group that contains property with adj list type.
+   *
+   * @param property_name name of the property.
+   * @param adj_list_type the type of adj list structure.
+   * @return property group that contains the property. If edge info not support the adj list type,
+   *         or not find the property group that contains the property, return false.
+   */
   def getPropertyGroup(property_name: String, adj_list_type: AdjListType.Value): PropertyGroup = {
     val tot: Int = adj_lists.size
     for ( k <- 0 to tot - 1 ) {
@@ -139,6 +183,12 @@ class EdgeInfo() {
     throw new IllegalArgumentException
   }
 
+  /** Get the data type of property.
+   *
+   * @param property_name name of the property.
+   * @return data type in gar of the property. If edge info not contains the property,
+   *         raise an IllegalArgumentException error.
+   */
   def getPropertyType(property_name: String): GarType.Value = {
     val tot: Int = adj_lists.size
     for ( k <- 0 to tot - 1 ) {
@@ -159,6 +209,12 @@ class EdgeInfo() {
     throw new IllegalArgumentException
   }
 
+  /** Check the property is primary key of edge info.
+   *
+   * @param property_name name of the property.
+   * @return true if the property is the primary key of edge info, false if not.
+   *         If edge info not contains the property, raise an IllegalArgumentException error.
+   */
   def isPrimaryKey(property_name: String): Boolean = {
     val tot: Int = adj_lists.size
     for ( k <- 0 to tot - 1 ) {
@@ -179,6 +235,7 @@ class EdgeInfo() {
     throw new IllegalArgumentException
   }
 
+  /** Get Primary key of edge info. */
   def getPrimaryKey(): String = {
     val tot: Int = adj_lists.size
     for ( k <- 0 to tot - 1 ) {
@@ -199,6 +256,7 @@ class EdgeInfo() {
     return ""
   }
 
+  /** Check if the edge info is validated. */
   def isValidated(): Boolean = {
     if (src_label == "" || edge_label == "" || dst_label == "")
       return false
@@ -222,6 +280,14 @@ class EdgeInfo() {
     return true
   }
 
+  /** Get the adj list offset chunk file path of vertex chunk
+   *  the offset chunks is aligned with the vertex chunks
+   *
+   *  @param chunk_inde index of vertex chunk.
+   *  @param adj_list_type type of adj list structure.
+   *  @return the offset chunk file path. If edge info not support the adj list type,
+   *          raise an IllegalArgumentException error.
+   */
   def getAdjListOffsetFilePath(chunk_index: Long, adj_list_type: AdjListType.Value) : String = {
     if (containAdjList(adj_list_type) == false)
       throw new IllegalArgumentException
@@ -230,28 +296,63 @@ class EdgeInfo() {
     return str
   }
 
+  /** Get the adj list offset chunk file directory path of adj list type.
+   *
+   * @param adj_list_type type of adj list structure.
+   * @return the offset directory. If edge info not support the adj list type,
+   *          raise an IllegalArgumentException error.
+   */
   def getAdjListOffsetDirPath(adj_list_type: AdjListType.Value) : String = {
     if (containAdjList(adj_list_type) == false)
       throw new IllegalArgumentException
     return prefix + getAdjListPrefix(adj_list_type) + "offset/"
   }
 
+  /** Get the file path of adj list topology chunk.
+   *
+   * @param vertex_chunk_index index of vertex chunk.
+   * @param chunk_index index of edge chunk.
+   * @param adj_list_type type of adj list structure.
+   * @return adj list chunk file path.
+   */
   def getAdjListFilePath(vertex_chunk_index: Long, chunk_index: Long, adj_list_type: AdjListType.Value) : String = {
     var str: String = prefix + getAdjListPrefix(adj_list_type) + "adj_list/part" +
         vertex_chunk_index.toString() + "/chunk" + chunk_index.toString()
     return str
   }
 
+  /** Get the path of adj list topology chunk of certain vertex chunk.
+   *
+   * @param vertex_chunk_index index of vertex chunk.
+   * @param adj_list_type type of adj list structure.
+   * @return path prefix of the edge chunk of vertices of given vertex chunk.
+   */
   def getAdjListFilePath(vertex_chunk_index: Long, adj_list_type: AdjListType.Value) : String = {
     var str: String = prefix + getAdjListPrefix(adj_list_type) + "adj_list/part" +
       vertex_chunk_index.toString() + "/"
     return str
   }
 
+  /** Get the adj list topology chunk file directory path of adj list type.
+   *
+   * @param adj_list_type type of adj list structure.
+   * @return directory path of adj list type.
+   */
   def getAdjListDirPath(adj_list_type: AdjListType.Value) : String = {
     return prefix + getAdjListPrefix(adj_list_type) + "adj_list/"
   }
 
+  /** Get the chunk file path of adj list property group.
+   *  the property group chunks is aligned with the adj list topology chunks
+   *
+   * @param property_group property group
+   * @param adj_list_type type of adj list structure.
+   * @param vertex_chunk_index index of vertex chunk.
+   * @param chunk_index index of edge chunk.
+   *
+   * @return property group chunk file path. If edge info not contains the property group,
+   *         raise an IllegalArgumentException error.
+   */
   def getPropertyFilePath(property_group: PropertyGroup, adj_list_type: AdjListType.Value, vertex_chunk_index: Long, chunk_index: Long) : String = {
     if (containPropertyGroup(property_group, adj_list_type) == false)
       throw new IllegalArgumentException
@@ -271,6 +372,15 @@ class EdgeInfo() {
     return str
   }
 
+  /** Get path of adj list property group of certain vertex chunk.
+   *
+   * @param property_group property group.
+   * @param adj_list_type type of adj list structure.
+   * @param vertex_chunk_index index of vertex chunk.
+   * @return path prefix of property group chunks of of vertices of given vertex chunk.
+   *         If edge info not contains the property group,
+   *         raise an IllegalArgumentException error.
+   */
   def getPropertyFilePath(property_group: PropertyGroup, adj_list_type: AdjListType.Value, vertex_chunk_index: Long) : String = {
     if (containPropertyGroup(property_group, adj_list_type) == false)
       throw new IllegalArgumentException
@@ -290,6 +400,13 @@ class EdgeInfo() {
     return str
   }
 
+  /** Get the property group chunk file directory path of adj list type.
+   *
+   * @param property_group property group.
+   * @param adj_list_type type of adj list structure.
+   * @return directory path of property group chunks. If edge info not contains the property group,
+   *         raise an IllegalArgumentException error.
+   */
   def getPropertyDirPath(property_group: PropertyGroup, adj_list_type: AdjListType.Value) : String = {
     if (containPropertyGroup(property_group, adj_list_type) == false)
       throw new IllegalArgumentException