From eb46bfbe455c80b0a2f60afd67be788ad647a99e Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:16:50 +0800 Subject: [PATCH] Revert "[HACKATHON 6th] move distributed unit tests (#62762)" (#62857) This reverts commit 67e02b0ab91f95199fe4682fb57771e9e3824c07. --- paddle/fluid/distributed/CMakeLists.txt | 1 + .../fluid/distributed/common/CMakeLists.txt | 2 + .../distributed/ps/service/CMakeLists.txt | 2 + .../fluid/distributed/test}/CMakeLists.txt | 100 ++++++++++++------ .../distributed/test}/barrier_table_test.cc | 0 .../test}/brpc_service_dense_sgd_test.cc | 0 .../test}/brpc_service_sparse_sgd_test.cc | 0 .../distributed/test}/brpc_utils_test.cc | 0 .../distributed/test}/ctr_accessor_test.cc | 0 .../test}/ctr_dymf_accessor_test.cc | 0 .../distributed/test}/dense_table_test.cc | 0 .../distributed/test}/feature_value_test.cc | 0 .../test}/graph_node_split_test.cc | 0 .../distributed/test}/graph_node_test.cc | 0 .../test}/graph_table_sample_test.cc | 0 .../test}/memory_geo_table_test.cc | 0 .../test}/memory_sparse_table_test.cc | 0 .../distributed/test}/sparse_sgd_rule_test.cc | 0 .../fluid/distributed/test}/table_test.cc | 0 test/cpp/fluid/CMakeLists.txt | 5 - test/cpp/fluid/pscore/CMakeLists.txt | 3 + 21 files changed, 74 insertions(+), 39 deletions(-) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/CMakeLists.txt (51%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/barrier_table_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/brpc_service_dense_sgd_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/brpc_service_sparse_sgd_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/brpc_utils_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/ctr_accessor_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/ctr_dymf_accessor_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/dense_table_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/feature_value_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/graph_node_split_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/graph_node_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/graph_table_sample_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/memory_geo_table_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/memory_sparse_table_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/sparse_sgd_rule_test.cc (100%) rename {test/cpp/fluid/distributed => paddle/fluid/distributed/test}/table_test.cc (100%) diff --git a/paddle/fluid/distributed/CMakeLists.txt b/paddle/fluid/distributed/CMakeLists.txt index f0347579cbbbb..f22e4d06ec78e 100755 --- a/paddle/fluid/distributed/CMakeLists.txt +++ b/paddle/fluid/distributed/CMakeLists.txt @@ -64,4 +64,5 @@ if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) endif() add_subdirectory(common) add_subdirectory(ps) +add_subdirectory(test) add_subdirectory(index_dataset) diff --git a/paddle/fluid/distributed/common/CMakeLists.txt b/paddle/fluid/distributed/common/CMakeLists.txt index 053ee2a349aab..fd738c274153f 100644 --- a/paddle/fluid/distributed/common/CMakeLists.txt +++ b/paddle/fluid/distributed/common/CMakeLists.txt @@ -2,3 +2,5 @@ cc_library( afs_wrapper SRCS afs_warpper.cc DEPS framework_io ps_framework_proto) + +#set_property(GLOBAL PROPERTY COMMON_DEPS afs_warpper) diff --git a/paddle/fluid/distributed/ps/service/CMakeLists.txt b/paddle/fluid/distributed/ps/service/CMakeLists.txt index 9f96eb6dba5af..eac2585416d8b 100755 --- a/paddle/fluid/distributed/ps/service/CMakeLists.txt +++ b/paddle/fluid/distributed/ps/service/CMakeLists.txt @@ -21,6 +21,8 @@ brpc_library( ps_framework_proto ${BRPC_DEPS}) +#set_property(GLOBAL PROPERTY RPC_DEPS sendrecv_rpc ${BRPC_DEPS} string_helper) + get_property(RPC_DEPS GLOBAL PROPERTY RPC_DEPS) proto_library(simple_brpc_proto SRCS simple_brpc.proto) diff --git a/test/cpp/fluid/distributed/CMakeLists.txt b/paddle/fluid/distributed/test/CMakeLists.txt similarity index 51% rename from test/cpp/fluid/distributed/CMakeLists.txt rename to paddle/fluid/distributed/test/CMakeLists.txt index 69411a5442977..ba08768ab4a10 100644 --- a/test/cpp/fluid/distributed/CMakeLists.txt +++ b/paddle/fluid/distributed/test/CMakeLists.txt @@ -1,84 +1,116 @@ -set(DISTRIBUTE_COMPILE_FLAGS - "-Wno-error=unused-value -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=parentheses -Wno-error=unused-result" -) - -if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) - set(DISTRIBUTE_COMPILE_FLAGS "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new") -endif() - -get_property(RPC_DEPS GLOBAL PROPERTY RPC_DEPS) - set_source_files_properties( table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(table_test SRCS table_test.cc DEPS ${RPC_DEPS}) +cc_test( + table_test + SRCS table_test.cc + DEPS common_table table ps_framework_proto ${COMMON_DEPS} ${RPC_DEPS}) set_source_files_properties( dense_table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(dense_table_test SRCS dense_table_test.cc DEPS ${RPC_DEPS}) +cc_test( + dense_table_test + SRCS dense_table_test.cc + DEPS common_table table ps_framework_proto ${COMMON_DEPS} ${RPC_DEPS}) set_source_files_properties( barrier_table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(barrier_table_test SRCS barrier_table_test.cc) +cc_test( + barrier_table_test + SRCS barrier_table_test.cc + DEPS common_table table ps_framework_proto ${COMMON_DEPS}) set_source_files_properties( brpc_service_dense_sgd_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(brpc_service_dense_sgd_test SRCS brpc_service_dense_sgd_test.cc - DEPS scope) +cc_test( + brpc_service_dense_sgd_test + SRCS brpc_service_dense_sgd_test.cc + DEPS scope ps_service table ps_framework_proto ${COMMON_DEPS}) set_source_files_properties( brpc_service_sparse_sgd_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(brpc_service_sparse_sgd_test SRCS brpc_service_sparse_sgd_test.cc - DEPS scope) +cc_test( + brpc_service_sparse_sgd_test + SRCS brpc_service_sparse_sgd_test.cc + DEPS scope ps_service table ps_framework_proto ${COMMON_DEPS}) set_source_files_properties( brpc_utils_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test( +cc_test( brpc_utils_test - SRCS - brpc_utils_test.cc - DEPS - scope - phi - common - ${RPC_DEPS}) + SRCS brpc_utils_test.cc + DEPS brpc_utils + scope + phi + common + sendrecv_rpc + ps_service + ${COMMON_DEPS} + ${RPC_DEPS}) set_source_files_properties( graph_node_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(graph_node_test SRCS graph_node_test.cc DEPS scope) +cc_test( + graph_node_test + SRCS graph_node_test.cc + DEPS scope ps_service table ps_framework_proto ${COMMON_DEPS}) set_source_files_properties( graph_node_split_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(graph_node_split_test SRCS graph_node_split_test.cc DEPS scope) +cc_test( + graph_node_split_test + SRCS graph_node_split_test.cc + DEPS scope ps_service table ps_framework_proto ${COMMON_DEPS}) set_source_files_properties( graph_table_sample_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(graph_table_sample_test SRCS graph_table_sample_test.cc) +cc_test( + graph_table_sample_test + SRCS graph_table_sample_test.cc + DEPS table ps_framework_proto ${COMMON_DEPS}) set_source_files_properties( feature_value_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(feature_value_test SRCS feature_value_test.cc) +cc_test( + feature_value_test + SRCS feature_value_test.cc + DEPS table common_table sendrecv_rpc ${COMMON_DEPS}) set_source_files_properties( sparse_sgd_rule_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(sparse_sgd_rule_test SRCS sparse_sgd_rule_test.cc) +cc_test( + sparse_sgd_rule_test + SRCS sparse_sgd_rule_test.cc + DEPS ${COMMON_DEPS} table) set_source_files_properties( ctr_accessor_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(ctr_accessor_test SRCS ctr_accessor_test.cc) +cc_test( + ctr_accessor_test + SRCS ctr_accessor_test.cc + DEPS ${COMMON_DEPS} table) set_source_files_properties( ctr_dymf_accessor_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(ctr_dymf_accessor_test SRCS ctr_dymf_accessor_test.cc) +cc_test( + ctr_dymf_accessor_test + SRCS ctr_dymf_accessor_test.cc + DEPS ${COMMON_DEPS} table) set_source_files_properties( memory_sparse_table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(memory_sparse_table_test SRCS memory_sparse_table_test.cc DEPS) +cc_test( + memory_sparse_table_test + SRCS memory_sparse_table_test.cc + DEPS ${COMMON_DEPS} table) set_source_files_properties( memory_geo_table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -paddle_test(memory_sparse_geo_table_test SRCS memory_geo_table_test.cc) +cc_test( + memory_sparse_geo_table_test + SRCS memory_geo_table_test.cc + DEPS ${COMMON_DEPS} table) diff --git a/test/cpp/fluid/distributed/barrier_table_test.cc b/paddle/fluid/distributed/test/barrier_table_test.cc similarity index 100% rename from test/cpp/fluid/distributed/barrier_table_test.cc rename to paddle/fluid/distributed/test/barrier_table_test.cc diff --git a/test/cpp/fluid/distributed/brpc_service_dense_sgd_test.cc b/paddle/fluid/distributed/test/brpc_service_dense_sgd_test.cc similarity index 100% rename from test/cpp/fluid/distributed/brpc_service_dense_sgd_test.cc rename to paddle/fluid/distributed/test/brpc_service_dense_sgd_test.cc diff --git a/test/cpp/fluid/distributed/brpc_service_sparse_sgd_test.cc b/paddle/fluid/distributed/test/brpc_service_sparse_sgd_test.cc similarity index 100% rename from test/cpp/fluid/distributed/brpc_service_sparse_sgd_test.cc rename to paddle/fluid/distributed/test/brpc_service_sparse_sgd_test.cc diff --git a/test/cpp/fluid/distributed/brpc_utils_test.cc b/paddle/fluid/distributed/test/brpc_utils_test.cc similarity index 100% rename from test/cpp/fluid/distributed/brpc_utils_test.cc rename to paddle/fluid/distributed/test/brpc_utils_test.cc diff --git a/test/cpp/fluid/distributed/ctr_accessor_test.cc b/paddle/fluid/distributed/test/ctr_accessor_test.cc similarity index 100% rename from test/cpp/fluid/distributed/ctr_accessor_test.cc rename to paddle/fluid/distributed/test/ctr_accessor_test.cc diff --git a/test/cpp/fluid/distributed/ctr_dymf_accessor_test.cc b/paddle/fluid/distributed/test/ctr_dymf_accessor_test.cc similarity index 100% rename from test/cpp/fluid/distributed/ctr_dymf_accessor_test.cc rename to paddle/fluid/distributed/test/ctr_dymf_accessor_test.cc diff --git a/test/cpp/fluid/distributed/dense_table_test.cc b/paddle/fluid/distributed/test/dense_table_test.cc similarity index 100% rename from test/cpp/fluid/distributed/dense_table_test.cc rename to paddle/fluid/distributed/test/dense_table_test.cc diff --git a/test/cpp/fluid/distributed/feature_value_test.cc b/paddle/fluid/distributed/test/feature_value_test.cc similarity index 100% rename from test/cpp/fluid/distributed/feature_value_test.cc rename to paddle/fluid/distributed/test/feature_value_test.cc diff --git a/test/cpp/fluid/distributed/graph_node_split_test.cc b/paddle/fluid/distributed/test/graph_node_split_test.cc similarity index 100% rename from test/cpp/fluid/distributed/graph_node_split_test.cc rename to paddle/fluid/distributed/test/graph_node_split_test.cc diff --git a/test/cpp/fluid/distributed/graph_node_test.cc b/paddle/fluid/distributed/test/graph_node_test.cc similarity index 100% rename from test/cpp/fluid/distributed/graph_node_test.cc rename to paddle/fluid/distributed/test/graph_node_test.cc diff --git a/test/cpp/fluid/distributed/graph_table_sample_test.cc b/paddle/fluid/distributed/test/graph_table_sample_test.cc similarity index 100% rename from test/cpp/fluid/distributed/graph_table_sample_test.cc rename to paddle/fluid/distributed/test/graph_table_sample_test.cc diff --git a/test/cpp/fluid/distributed/memory_geo_table_test.cc b/paddle/fluid/distributed/test/memory_geo_table_test.cc similarity index 100% rename from test/cpp/fluid/distributed/memory_geo_table_test.cc rename to paddle/fluid/distributed/test/memory_geo_table_test.cc diff --git a/test/cpp/fluid/distributed/memory_sparse_table_test.cc b/paddle/fluid/distributed/test/memory_sparse_table_test.cc similarity index 100% rename from test/cpp/fluid/distributed/memory_sparse_table_test.cc rename to paddle/fluid/distributed/test/memory_sparse_table_test.cc diff --git a/test/cpp/fluid/distributed/sparse_sgd_rule_test.cc b/paddle/fluid/distributed/test/sparse_sgd_rule_test.cc similarity index 100% rename from test/cpp/fluid/distributed/sparse_sgd_rule_test.cc rename to paddle/fluid/distributed/test/sparse_sgd_rule_test.cc diff --git a/test/cpp/fluid/distributed/table_test.cc b/paddle/fluid/distributed/test/table_test.cc similarity index 100% rename from test/cpp/fluid/distributed/table_test.cc rename to paddle/fluid/distributed/test/table_test.cc diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt index 0b249c4adc252..3a8f9326764cb 100644 --- a/test/cpp/fluid/CMakeLists.txt +++ b/test/cpp/fluid/CMakeLists.txt @@ -21,11 +21,6 @@ if(WITH_MKLDNN) add_subdirectory(mkldnn) endif() add_subdirectory(nccl) - -if(WITH_DISTRIBUTE) - add_subdirectory(distributed) -endif() - if(WITH_PSCORE) add_subdirectory(pscore) endif() diff --git a/test/cpp/fluid/pscore/CMakeLists.txt b/test/cpp/fluid/pscore/CMakeLists.txt index 3b74fd0a6f793..c95841199d76b 100644 --- a/test/cpp/fluid/pscore/CMakeLists.txt +++ b/test/cpp/fluid/pscore/CMakeLists.txt @@ -67,6 +67,9 @@ set_source_files_properties( ${DISTRIBUTE_COMPILE_FLAGS}) paddle_test(heter_listen_and_server_test SRCS heter_listen_and_server_test.cc) +#set_source_files_properties(heter_cloud_comm_cpu_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) +#cc_test(heter_cloud_comm_cpu_test SRCS heter_cloud_comm_cpu_test.cc DEPS executor scope proto_desc generated_static_op heter_listen_and_serv_op ${RPC_DEPS} ${DISTRIBUTE_DEPS} phi common) + set_source_files_properties( switch_server_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) paddle_test(switch_server_test SRCS switch_server_test.cc)