Feat(CI): Use environment variable to specify the location of testing…

… data Signed-off-by: acezen <[email protected]> Fix the format and lint Signed-off-by: acezen <[email protected]>
apache · Jun 4, 2024 · 79e7cff · 79e7cff
1 parent 82a02f1
commit 79e7cff
Show file tree

Hide file tree

Showing 38 changed files with 252 additions and 310 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -42,7 +42,7 @@ jobs:
     name: Ubuntu 22.04 C++
     runs-on: ubuntu-latest
     env:
-      GAR_TEST_DATA: ${{ github.workspace }}/testing/
+      GAR_TEST_DATA: ${{ github.workspace }}/graphar-testing/
     steps:
     - uses: actions/checkout@v3
       with:
@@ -76,6 +76,8 @@ jobs:
         sudo cmake --build build/ --target install
         popd
 
+        git clone https://github.com/apache/incubator-graphar-testing.git $GAR_TEST_DATA --depth 1
+
     - name: CMake
       working-directory: "cpp"
       run: |

diff --git a/.github/workflows/spark.yaml b/.github/workflows/spark.yaml
@@ -40,6 +40,8 @@ concurrency:
 jobs:
   test:
     runs-on: ubuntu-20.04
+    env:
+      GAR_TEST_DATA: ${{ github.workspace }}/graphar-testing/
     strategy:
       fail-fast: false
       matrix:
@@ -62,6 +64,10 @@ jobs:
         export JAVA_HOME=${JAVA_HOME_11_X64}
         mvn --no-transfer-progress spotless:check
 
+    - name: Download test data
+      run: |
+        git clone https://github.com/apache/incubator-graphar-testing.git $GAR_TEST_DATA --depth 1
+
     - name: Build GraphAr Spark
       working-directory: maven-projects/spark
       run: |

diff --git a/cpp/README.md b/cpp/README.md
@@ -88,21 +88,27 @@ Debug build with unit tests:
     $ cd build-debug
     $ cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON ..
     $ make -j8       # if you have 8 CPU cores, otherwise adjust, use -j`nproc` for all cores
-    $ make test      # to run the tests
+```
+
+After building, you can run the unit tests with:
+
+```bash
+    $ git clone https://github.com/apache/incubator-graphar-testing.git testing  # download the testing data
+    $ GAR_TEST_DATA=${PWD}/testing ctest
 ```
 
 Build with examples, you should build the project with `BUILD_EXAMPLES` option, then run:
 
 ```bash
     $ make -j8       # if you have 8 CPU cores, otherwise adjust, use -j`nproc` for all cores
-    $ ./bgl_example  # run the BGL example
+    $ GAR_TEST_DATA=${PWD}/testing ./bgl_example  # run the BGL example
 ```
 
 Build with benchmarks, you should build the project with `BUILD_BENCHMARKS` option, then run:
 
 ```bash
     $ make -j8       # if you have 8 CPU cores, otherwise adjust, use -j`nproc` for all cores
-    $ ./graph_info_benchmark  # run the graph info benchmark
+    $ GAR_TEST_DATA=${PWD}/testing ./graph_info_benchmark  # run the graph info benchmark
 ```
 
 ### Install

diff --git a/cpp/benchmarks/benchmark_util.h b/cpp/benchmarks/benchmark_util.h
@@ -30,19 +30,16 @@
 
 namespace graphar {
 
-static const std::string TEST_DATA_DIR =  // NOLINT
-    std::filesystem::path(__FILE__)
-        .parent_path()
-        .parent_path()
-        .parent_path()
-        .parent_path()
-        .string() +
-    "/testing";
-
 class BenchmarkFixture : public ::benchmark::Fixture {
  public:
   void SetUp(const ::benchmark::State& state) override {
-    path_ = TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+    const char* c_root = std::getenv("GAR_TEST_DATA");
+    if (!c_root) {
+      throw std::runtime_error(
+          "Test resources not found, set GAR_TEST_DATA to auxiliary testing "
+          "data");
+    }
+    path_ = std::string(c_root) + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
     auto maybe_graph_info = GraphInfo::Load(path_);
     graph_info_ = maybe_graph_info.value();
   }

diff --git a/cpp/examples/bfs_father_example.cc b/cpp/examples/bfs_father_example.cc
@@ -31,7 +31,7 @@
 int main(int argc, char* argv[]) {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   auto graph_info = graphar::GraphInfo::Load(path).value();
 
   // get the person vertices of graph

diff --git a/cpp/examples/bfs_pull_example.cc b/cpp/examples/bfs_pull_example.cc
@@ -30,7 +30,7 @@
 int main(int argc, char* argv[]) {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   auto graph_info = graphar::GraphInfo::Load(path).value();
 
   // construct vertices collection

diff --git a/cpp/examples/bfs_push_example.cc b/cpp/examples/bfs_push_example.cc
@@ -30,7 +30,7 @@
 int main(int argc, char* argv[]) {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   auto graph_info = graphar::GraphInfo::Load(path).value();
 
   // construct vertices collection

diff --git a/cpp/examples/bfs_stream_example.cc b/cpp/examples/bfs_stream_example.cc
@@ -30,7 +30,7 @@
 int main(int argc, char* argv[]) {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   auto graph_info = graphar::GraphInfo::Load(path).value();
 
   // construct vertices collection

diff --git a/cpp/examples/bgl_example.cc b/cpp/examples/bgl_example.cc
@@ -35,7 +35,7 @@
 int main(int argc, char* argv[]) {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   auto graph_info = graphar::GraphInfo::Load(path).value();
   ASSERT(graph_info->GetVertexInfos().size() == 1);
   ASSERT(graph_info->GetEdgeInfos().size() == 1);

diff --git a/cpp/examples/cc_push_example.cc b/cpp/examples/cc_push_example.cc
@@ -31,7 +31,7 @@
 int main(int argc, char* argv[]) {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   auto graph_info = graphar::GraphInfo::Load(path).value();
 
   // construct vertices collection

diff --git a/cpp/examples/cc_stream_example.cc b/cpp/examples/cc_stream_example.cc
@@ -31,7 +31,7 @@
 int main(int argc, char* argv[]) {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   auto graph_info = graphar::GraphInfo::Load(path).value();
 
   // construct vertices collection

diff --git a/cpp/examples/config.h b/cpp/examples/config.h
@@ -56,11 +56,12 @@
 #define DASSERT(x)
 #endif
 
-static const std::string TEST_DATA_DIR =  // NOLINT
-    std::filesystem::path(__FILE__)
-        .parent_path()
-        .parent_path()
-        .parent_path()
-        .parent_path()
-        .string() +
-    "/testing";
+std::string GetTestingResourceRoot() {
+  const char* c_root = std::getenv("GAR_TEST_DATA");
+  if (!c_root) {
+    throw std::runtime_error(
+        "Test resources not found, set GAR_TEST_DATA to auxiliary testing "
+        "data");
+  }
+  return std::string(c_root);
+}
diff --git a/cpp/examples/high_level_reader_example.cc b/cpp/examples/high_level_reader_example.cc
@@ -120,7 +120,7 @@ void edges_collection(const std::shared_ptr<graphar::GraphInfo>& graph_info) {
 int main(int argc, char* argv[]) {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   auto graph_info = graphar::GraphInfo::Load(path).value();
 
   // vertices collection

diff --git a/cpp/examples/high_level_writer_example.cc b/cpp/examples/high_level_writer_example.cc
@@ -30,7 +30,7 @@
 void vertices_builder() {
   // construct vertices builder
   std::string vertex_meta_file =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/" + "person.vertex.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/" + "person.vertex.yml";
   auto vertex_meta = graphar::Yaml::LoadFile(vertex_meta_file).value();
   auto vertex_info = graphar::VertexInfo::Load(vertex_meta).value();
   graphar::IdType start_index = 0;
@@ -71,8 +71,9 @@ void vertices_builder() {
 
 void edges_builder() {
   // construct edges builder
-  std::string edge_meta_file =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/" + "person_knows_person.edge.yml";
+  std::string edge_meta_file = GetTestingResourceRoot() +
+                               "/ldbc_sample/parquet/" +
+                               "person_knows_person.edge.yml";
   auto edge_meta = graphar::Yaml::LoadFile(edge_meta_file).value();
   auto edge_info = graphar::EdgeInfo::Load(edge_meta).value();
   auto vertex_count = 3;

diff --git a/cpp/examples/low_level_reader_example.cc b/cpp/examples/low_level_reader_example.cc
@@ -128,7 +128,7 @@ void adj_list_property_chunk_info_reader(
 int main(int argc, char* argv[]) {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   auto graph_info = graphar::GraphInfo::Load(path).value();
 
   // vertex property chunk info reader

diff --git a/cpp/examples/mid_level_reader_example.cc b/cpp/examples/mid_level_reader_example.cc
@@ -215,7 +215,7 @@ void adj_list_offset_chunk_reader(
 int main(int argc, char* argv[]) {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   auto graph_info = graphar::GraphInfo::Load(path).value();
 
   // vertex property chunk reader

diff --git a/cpp/examples/mid_level_writer_example.cc b/cpp/examples/mid_level_writer_example.cc
@@ -105,7 +105,7 @@ void vertex_property_writer(
     const std::shared_ptr<graphar::GraphInfo>& graph_info) {
   // create writer
   std::string vertex_meta_file =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/" + "person.vertex.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/" + "person.vertex.yml";
   auto vertex_meta = graphar::Yaml::LoadFile(vertex_meta_file).value();
   auto vertex_info = graphar::VertexInfo::Load(vertex_meta).value();
   ASSERT(vertex_info->GetLabel() == "person");
@@ -141,8 +141,8 @@ void vertex_property_writer(
 
 void edge_chunk_writer(const std::shared_ptr<graphar::GraphInfo>& graph_info) {
   // construct writer
-  std::string edge_meta_file =
-      TEST_DATA_DIR + "/ldbc_sample/csv/" + "person_knows_person.edge.yml";
+  std::string edge_meta_file = GetTestingResourceRoot() + "/ldbc_sample/csv/" +
+                               "person_knows_person.edge.yml";
   auto edge_meta = graphar::Yaml::LoadFile(edge_meta_file).value();
   auto edge_info = graphar::EdgeInfo::Load(edge_meta).value();
   auto adj_list_type = graphar::AdjListType::ordered_by_source;
@@ -194,7 +194,7 @@ void edge_chunk_writer(const std::shared_ptr<graphar::GraphInfo>& graph_info) {
 int main(int argc, char* argv[]) {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   auto graph_info = graphar::GraphInfo::Load(path).value();
 
   // vertex property writer

diff --git a/cpp/examples/pagerank_example.cc b/cpp/examples/pagerank_example.cc
@@ -31,7 +31,7 @@
 int main(int argc, char* argv[]) {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      GetTestingResourceRoot() + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   auto graph_info = graphar::GraphInfo::Load(path).value();
 
   // construct vertices collection

diff --git a/cpp/test/test_arrow_chunk_reader.cc b/cpp/test/test_arrow_chunk_reader.cc
@@ -32,10 +32,10 @@
 #include <catch2/catch_test_macros.hpp>
 namespace graphar {
 
-TEST_CASE("ArrowChunkReader") {
+TEST_CASE_METHOD(GlobalFixture, "ArrowChunkReader") {
   // read file and construct graph info
   std::string path =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
+      test_data_dir + "/ldbc_sample/parquet/ldbc_sample.graph.yml";
   std::string src_label = "person", edge_label = "knows", dst_label = "person";
   std::string vertex_property_name = "id";
   std::string edge_property_name = "creationDate";
@@ -95,7 +95,7 @@ TEST_CASE("ArrowChunkReader") {
     }
 
     SECTION("CastDataType") {
-      std::string prefix = TEST_DATA_DIR + "/modern_graph/";
+      std::string prefix = test_data_dir + "/modern_graph/";
       std::string vertex_info_path = prefix + "person.vertex.yml";
       std::cout << "Vertex info path: " << vertex_info_path << std::endl;
       auto fs = FileSystemFromUriOrPath(prefix).value();

diff --git a/cpp/test/test_arrow_chunk_writer.cc b/cpp/test/test_arrow_chunk_writer.cc
@@ -43,8 +43,13 @@
 
 namespace graphar {
 
+<<<<<<< HEAD
 TEST_CASE("TestVertexPropertyWriter") {
   std::string path = TEST_DATA_DIR + "/ldbc_sample/person_0_0.csv";
+=======
+TEST_CASE_METHOD(GlobalFixture, "test_vertex_property_writer_from_file") {
+  std::string path = test_data_dir + "/ldbc_sample/person_0_0.csv";
+>>>>>>> 9fd665b (Feat(CI): Use  environment variable to specify the location of testing data)
   arrow::io::IOContext io_context = arrow::io::default_io_context();
 
   auto fs = arrow::fs::FileSystemFromUriOrPath(path).ValueOrDie();
@@ -70,7 +75,7 @@ TEST_CASE("TestVertexPropertyWriter") {
 
   // Construct the writer
   std::string vertex_meta_file =
-      TEST_DATA_DIR + "/ldbc_sample/parquet/" + "person.vertex.yml";
+      test_data_dir + "/ldbc_sample/parquet/" + "person.vertex.yml";
   auto vertex_meta = Yaml::LoadFile(vertex_meta_file).value();
   auto vertex_info = VertexInfo::Load(vertex_meta).value();
   auto maybe_writer = VertexPropertyWriter::Make(vertex_info, "/tmp/");
@@ -119,9 +124,9 @@ TEST_CASE("TestVertexPropertyWriter") {
   SECTION("TestOrcParquetReader") {
     arrow::Status st;
     arrow::MemoryPool* pool = arrow::default_memory_pool();
-    std::string path1 = TEST_DATA_DIR + "/ldbc_sample/orc" +
+    std::string path1 = test_data_dir + "/ldbc_sample/orc" +
                         "/vertex/person/firstName_lastName_gender/chunk1";
-    std::string path2 = TEST_DATA_DIR + "/ldbc_sample/parquet" +
+    std::string path2 = test_data_dir + "/ldbc_sample/parquet" +
                         "/vertex/person/firstName_lastName_gender/chunk1";
     arrow::io::IOContext io_context = arrow::io::default_io_context();
 
@@ -158,7 +163,7 @@ TEST_CASE("TestVertexPropertyWriter") {
   SECTION("TestEdgeChunkWriter") {
     arrow::Status st;
     arrow::MemoryPool* pool = arrow::default_memory_pool();
-    std::string path = TEST_DATA_DIR +
+    std::string path = test_data_dir +
                        "/ldbc_sample/parquet/edge/person_knows_person/"
                        "unordered_by_source/adj_list/part0/chunk0";
     auto fs = arrow::fs::FileSystemFromUriOrPath(path).ValueOrDie();
@@ -181,7 +186,7 @@ TEST_CASE("TestVertexPropertyWriter") {
 
     // Construct the writer
     std::string edge_meta_file =
-        TEST_DATA_DIR + "/ldbc_sample/csv/" + "person_knows_person.edge.yml";
+        test_data_dir + "/ldbc_sample/csv/" + "person_knows_person.edge.yml";
     auto edge_meta = Yaml::LoadFile(edge_meta_file).value();
     auto edge_info = EdgeInfo::Load(edge_meta).value();
     auto adj_list_type = AdjListType::ordered_by_source;