Revert "[HACKATHON 6th] move distributed unit tests (#62762)" (#62857)

This reverts commit 67e02b0.
PaddlePaddle · Mar 20, 2024 · eb46bfb · eb46bfb
1 parent f962c9d
commit eb46bfb
Show file tree

Hide file tree

Showing 21 changed files with 74 additions and 39 deletions.
diff --git a/paddle/fluid/distributed/CMakeLists.txt b/paddle/fluid/distributed/CMakeLists.txt
@@ -64,4 +64,5 @@ if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
 endif()
 add_subdirectory(common)
 add_subdirectory(ps)
+add_subdirectory(test)
 add_subdirectory(index_dataset)
diff --git a/paddle/fluid/distributed/common/CMakeLists.txt b/paddle/fluid/distributed/common/CMakeLists.txt
@@ -2,3 +2,5 @@ cc_library(
   afs_wrapper
   SRCS afs_warpper.cc
   DEPS framework_io ps_framework_proto)
+
+#set_property(GLOBAL PROPERTY COMMON_DEPS afs_warpper)
diff --git a/paddle/fluid/distributed/ps/service/CMakeLists.txt b/paddle/fluid/distributed/ps/service/CMakeLists.txt
@@ -21,6 +21,8 @@ brpc_library(
   ps_framework_proto
   ${BRPC_DEPS})
 
+#set_property(GLOBAL PROPERTY RPC_DEPS sendrecv_rpc ${BRPC_DEPS} string_helper)
+
 get_property(RPC_DEPS GLOBAL PROPERTY RPC_DEPS)
 
 proto_library(simple_brpc_proto SRCS simple_brpc.proto)

diff --git a/test/cpp/fluid/distributed/CMakeLists.txt → paddle/fluid/distributed/test/CMakeLists.txt b/test/cpp/fluid/distributed/CMakeLists.txt → paddle/fluid/distributed/test/CMakeLists.txt
@@ -1,84 +1,116 @@
-set(DISTRIBUTE_COMPILE_FLAGS
-    "-Wno-error=unused-value -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=parentheses -Wno-error=unused-result"
-)
-
-if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
-  set(DISTRIBUTE_COMPILE_FLAGS "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new")
-endif()
-
-get_property(RPC_DEPS GLOBAL PROPERTY RPC_DEPS)
-
 set_source_files_properties(
   table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(table_test SRCS table_test.cc DEPS ${RPC_DEPS})
+cc_test(
+  table_test
+  SRCS table_test.cc
+  DEPS common_table table ps_framework_proto ${COMMON_DEPS} ${RPC_DEPS})
 
 set_source_files_properties(
   dense_table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(dense_table_test SRCS dense_table_test.cc DEPS ${RPC_DEPS})
+cc_test(
+  dense_table_test
+  SRCS dense_table_test.cc
+  DEPS common_table table ps_framework_proto ${COMMON_DEPS} ${RPC_DEPS})
 
 set_source_files_properties(
   barrier_table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(barrier_table_test SRCS barrier_table_test.cc)
+cc_test(
+  barrier_table_test
+  SRCS barrier_table_test.cc
+  DEPS common_table table ps_framework_proto ${COMMON_DEPS})
 
 set_source_files_properties(
   brpc_service_dense_sgd_test.cc PROPERTIES COMPILE_FLAGS
                                             ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(brpc_service_dense_sgd_test SRCS brpc_service_dense_sgd_test.cc
-            DEPS scope)
+cc_test(
+  brpc_service_dense_sgd_test
+  SRCS brpc_service_dense_sgd_test.cc
+  DEPS scope ps_service table ps_framework_proto ${COMMON_DEPS})
 
 set_source_files_properties(
   brpc_service_sparse_sgd_test.cc PROPERTIES COMPILE_FLAGS
                                              ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(brpc_service_sparse_sgd_test SRCS brpc_service_sparse_sgd_test.cc
-            DEPS scope)
+cc_test(
+  brpc_service_sparse_sgd_test
+  SRCS brpc_service_sparse_sgd_test.cc
+  DEPS scope ps_service table ps_framework_proto ${COMMON_DEPS})
 
 set_source_files_properties(
   brpc_utils_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(
+cc_test(
   brpc_utils_test
-  SRCS
-  brpc_utils_test.cc
-  DEPS
-  scope
-  phi
-  common
-  ${RPC_DEPS})
+  SRCS brpc_utils_test.cc
+  DEPS brpc_utils
+       scope
+       phi
+       common
+       sendrecv_rpc
+       ps_service
+       ${COMMON_DEPS}
+       ${RPC_DEPS})
 
 set_source_files_properties(
   graph_node_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(graph_node_test SRCS graph_node_test.cc DEPS scope)
+cc_test(
+  graph_node_test
+  SRCS graph_node_test.cc
+  DEPS scope ps_service table ps_framework_proto ${COMMON_DEPS})
 
 set_source_files_properties(
   graph_node_split_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(graph_node_split_test SRCS graph_node_split_test.cc DEPS scope)
+cc_test(
+  graph_node_split_test
+  SRCS graph_node_split_test.cc
+  DEPS scope ps_service table ps_framework_proto ${COMMON_DEPS})
 
 set_source_files_properties(
   graph_table_sample_test.cc PROPERTIES COMPILE_FLAGS
                                         ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(graph_table_sample_test SRCS graph_table_sample_test.cc)
+cc_test(
+  graph_table_sample_test
+  SRCS graph_table_sample_test.cc
+  DEPS table ps_framework_proto ${COMMON_DEPS})
 
 set_source_files_properties(
   feature_value_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
 
-paddle_test(feature_value_test SRCS feature_value_test.cc)
+cc_test(
+  feature_value_test
+  SRCS feature_value_test.cc
+  DEPS table common_table sendrecv_rpc ${COMMON_DEPS})
 
 set_source_files_properties(
   sparse_sgd_rule_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(sparse_sgd_rule_test SRCS sparse_sgd_rule_test.cc)
+cc_test(
+  sparse_sgd_rule_test
+  SRCS sparse_sgd_rule_test.cc
+  DEPS ${COMMON_DEPS} table)
 
 set_source_files_properties(
   ctr_accessor_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(ctr_accessor_test SRCS ctr_accessor_test.cc)
+cc_test(
+  ctr_accessor_test
+  SRCS ctr_accessor_test.cc
+  DEPS ${COMMON_DEPS} table)
 set_source_files_properties(
   ctr_dymf_accessor_test.cc PROPERTIES COMPILE_FLAGS
                                        ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(ctr_dymf_accessor_test SRCS ctr_dymf_accessor_test.cc)
+cc_test(
+  ctr_dymf_accessor_test
+  SRCS ctr_dymf_accessor_test.cc
+  DEPS ${COMMON_DEPS} table)
 
 set_source_files_properties(
   memory_sparse_table_test.cc PROPERTIES COMPILE_FLAGS
                                          ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(memory_sparse_table_test SRCS memory_sparse_table_test.cc DEPS)
+cc_test(
+  memory_sparse_table_test
+  SRCS memory_sparse_table_test.cc
+  DEPS ${COMMON_DEPS} table)
 
 set_source_files_properties(
   memory_geo_table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
-paddle_test(memory_sparse_geo_table_test SRCS memory_geo_table_test.cc)
+cc_test(
+  memory_sparse_geo_table_test
+  SRCS memory_geo_table_test.cc
+  DEPS ${COMMON_DEPS} table)
diff --git a/...p/fluid/distributed/barrier_table_test.cc → ...id/distributed/test/barrier_table_test.cc b/...p/fluid/distributed/barrier_table_test.cc → ...id/distributed/test/barrier_table_test.cc
diff --git a/...istributed/brpc_service_dense_sgd_test.cc → ...buted/test/brpc_service_dense_sgd_test.cc b/...istributed/brpc_service_dense_sgd_test.cc → ...buted/test/brpc_service_dense_sgd_test.cc
diff --git a/...stributed/brpc_service_sparse_sgd_test.cc → ...uted/test/brpc_service_sparse_sgd_test.cc b/...stributed/brpc_service_sparse_sgd_test.cc → ...uted/test/brpc_service_sparse_sgd_test.cc
diff --git a/.../cpp/fluid/distributed/brpc_utils_test.cc → ...fluid/distributed/test/brpc_utils_test.cc b/.../cpp/fluid/distributed/brpc_utils_test.cc → ...fluid/distributed/test/brpc_utils_test.cc
diff --git a/...pp/fluid/distributed/ctr_accessor_test.cc → ...uid/distributed/test/ctr_accessor_test.cc b/...pp/fluid/distributed/ctr_accessor_test.cc → ...uid/distributed/test/ctr_accessor_test.cc
diff --git a/...uid/distributed/ctr_dymf_accessor_test.cc → ...istributed/test/ctr_dymf_accessor_test.cc b/...uid/distributed/ctr_dymf_accessor_test.cc → ...istributed/test/ctr_dymf_accessor_test.cc
diff --git a/...cpp/fluid/distributed/dense_table_test.cc → ...luid/distributed/test/dense_table_test.cc b/...cpp/fluid/distributed/dense_table_test.cc → ...luid/distributed/test/dense_table_test.cc
diff --git a/...p/fluid/distributed/feature_value_test.cc → ...id/distributed/test/feature_value_test.cc b/...p/fluid/distributed/feature_value_test.cc → ...id/distributed/test/feature_value_test.cc
diff --git a/...luid/distributed/graph_node_split_test.cc → ...distributed/test/graph_node_split_test.cc b/...luid/distributed/graph_node_split_test.cc → ...distributed/test/graph_node_split_test.cc
diff --git a/.../cpp/fluid/distributed/graph_node_test.cc → ...fluid/distributed/test/graph_node_test.cc b/.../cpp/fluid/distributed/graph_node_test.cc → ...fluid/distributed/test/graph_node_test.cc
diff --git a/...id/distributed/graph_table_sample_test.cc → ...stributed/test/graph_table_sample_test.cc b/...id/distributed/graph_table_sample_test.cc → ...stributed/test/graph_table_sample_test.cc
diff --git a/...luid/distributed/memory_geo_table_test.cc → ...distributed/test/memory_geo_table_test.cc b/...luid/distributed/memory_geo_table_test.cc → ...distributed/test/memory_geo_table_test.cc
diff --git a/...d/distributed/memory_sparse_table_test.cc → ...tributed/test/memory_sparse_table_test.cc b/...d/distributed/memory_sparse_table_test.cc → ...tributed/test/memory_sparse_table_test.cc
diff --git a/...fluid/distributed/sparse_sgd_rule_test.cc → .../distributed/test/sparse_sgd_rule_test.cc b/...fluid/distributed/sparse_sgd_rule_test.cc → .../distributed/test/sparse_sgd_rule_test.cc
diff --git a/test/cpp/fluid/distributed/table_test.cc → paddle/fluid/distributed/test/table_test.cc b/test/cpp/fluid/distributed/table_test.cc → paddle/fluid/distributed/test/table_test.cc
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
@@ -21,11 +21,6 @@ if(WITH_MKLDNN)
   add_subdirectory(mkldnn)
 endif()
 add_subdirectory(nccl)
-
-if(WITH_DISTRIBUTE)
-  add_subdirectory(distributed)
-endif()
-
 if(WITH_PSCORE)
   add_subdirectory(pscore)
 endif()

diff --git a/test/cpp/fluid/pscore/CMakeLists.txt b/test/cpp/fluid/pscore/CMakeLists.txt
@@ -67,6 +67,9 @@ set_source_files_properties(
                                              ${DISTRIBUTE_COMPILE_FLAGS})
 paddle_test(heter_listen_and_server_test SRCS heter_listen_and_server_test.cc)
 
+#set_source_files_properties(heter_cloud_comm_cpu_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
+#cc_test(heter_cloud_comm_cpu_test SRCS heter_cloud_comm_cpu_test.cc DEPS executor scope proto_desc generated_static_op heter_listen_and_serv_op ${RPC_DEPS} ${DISTRIBUTE_DEPS} phi common)
+
 set_source_files_properties(
   switch_server_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
 paddle_test(switch_server_test SRCS switch_server_test.cc)