From f4a7ae8f7a3c3f598afd450242c6f4342a6dcca5 Mon Sep 17 00:00:00 2001 From: Silver Ling Date: Fri, 15 Mar 2024 11:49:05 +0000 Subject: [PATCH] [HACKATHON 6th] move distributed unit tests --- paddle/fluid/distributed/CMakeLists.txt | 1 - .../fluid/distributed/common/CMakeLists.txt | 2 - .../distributed/ps/service/CMakeLists.txt | 2 - test/cpp/fluid/CMakeLists.txt | 5 + .../cpp/fluid/distributed}/CMakeLists.txt | 150 +++++++++++------- .../fluid/distributed}/barrier_table_test.cc | 0 .../brpc_service_dense_sgd_test.cc | 0 .../brpc_service_sparse_sgd_test.cc | 0 .../cpp/fluid/distributed}/brpc_utils_test.cc | 0 .../fluid/distributed}/ctr_accessor_test.cc | 0 .../distributed}/ctr_dymf_accessor_test.cc | 0 .../fluid/distributed}/dense_table_test.cc | 0 .../fluid/distributed}/feature_value_test.cc | 0 .../distributed}/graph_node_split_test.cc | 0 .../cpp/fluid/distributed}/graph_node_test.cc | 0 .../distributed}/graph_table_sample_test.cc | 0 .../distributed}/memory_geo_table_test.cc | 0 .../distributed}/memory_sparse_table_test.cc | 0 .../distributed}/sparse_sgd_rule_test.cc | 0 .../cpp/fluid/distributed}/table_test.cc | 0 test/cpp/fluid/pscore/CMakeLists.txt | 3 - 21 files changed, 97 insertions(+), 66 deletions(-) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/CMakeLists.txt (52%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/barrier_table_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/brpc_service_dense_sgd_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/brpc_service_sparse_sgd_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/brpc_utils_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/ctr_accessor_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/ctr_dymf_accessor_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/dense_table_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/feature_value_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/graph_node_split_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/graph_node_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/graph_table_sample_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/memory_geo_table_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/memory_sparse_table_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/sparse_sgd_rule_test.cc (100%) rename {paddle/fluid/distributed/test => test/cpp/fluid/distributed}/table_test.cc (100%) diff --git a/paddle/fluid/distributed/CMakeLists.txt b/paddle/fluid/distributed/CMakeLists.txt index f22e4d06ec78e..f0347579cbbbb 100755 --- a/paddle/fluid/distributed/CMakeLists.txt +++ b/paddle/fluid/distributed/CMakeLists.txt @@ -64,5 +64,4 @@ if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) endif() add_subdirectory(common) add_subdirectory(ps) -add_subdirectory(test) add_subdirectory(index_dataset) diff --git a/paddle/fluid/distributed/common/CMakeLists.txt b/paddle/fluid/distributed/common/CMakeLists.txt index fd738c274153f..053ee2a349aab 100644 --- a/paddle/fluid/distributed/common/CMakeLists.txt +++ b/paddle/fluid/distributed/common/CMakeLists.txt @@ -2,5 +2,3 @@ cc_library( afs_wrapper SRCS afs_warpper.cc DEPS framework_io ps_framework_proto) - -#set_property(GLOBAL PROPERTY COMMON_DEPS afs_warpper) diff --git a/paddle/fluid/distributed/ps/service/CMakeLists.txt b/paddle/fluid/distributed/ps/service/CMakeLists.txt index eac2585416d8b..9f96eb6dba5af 100755 --- a/paddle/fluid/distributed/ps/service/CMakeLists.txt +++ b/paddle/fluid/distributed/ps/service/CMakeLists.txt @@ -21,8 +21,6 @@ brpc_library( ps_framework_proto ${BRPC_DEPS}) -#set_property(GLOBAL PROPERTY RPC_DEPS sendrecv_rpc ${BRPC_DEPS} string_helper) - get_property(RPC_DEPS GLOBAL PROPERTY RPC_DEPS) proto_library(simple_brpc_proto SRCS simple_brpc.proto) diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt index 3a8f9326764cb..0b249c4adc252 100644 --- a/test/cpp/fluid/CMakeLists.txt +++ b/test/cpp/fluid/CMakeLists.txt @@ -21,6 +21,11 @@ if(WITH_MKLDNN) add_subdirectory(mkldnn) endif() add_subdirectory(nccl) + +if(WITH_DISTRIBUTE) + add_subdirectory(distributed) +endif() + if(WITH_PSCORE) add_subdirectory(pscore) endif() diff --git a/paddle/fluid/distributed/test/CMakeLists.txt b/test/cpp/fluid/distributed/CMakeLists.txt similarity index 52% rename from paddle/fluid/distributed/test/CMakeLists.txt rename to test/cpp/fluid/distributed/CMakeLists.txt index ba08768ab4a10..4ef4a10a0da52 100644 --- a/paddle/fluid/distributed/test/CMakeLists.txt +++ b/test/cpp/fluid/distributed/CMakeLists.txt @@ -1,116 +1,150 @@ +set(DISTRIBUTE_COMPILE_FLAGS + "-Wno-error=unused-value -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=parentheses -Wno-error=unused-result" +) + +if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) + set(DISTRIBUTE_COMPILE_FLAGS "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new") +endif() + +get_property(RPC_DEPS GLOBAL PROPERTY RPC_DEPS) + set_source_files_properties( table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( +paddle_test( table_test - SRCS table_test.cc - DEPS common_table table ps_framework_proto ${COMMON_DEPS} ${RPC_DEPS}) + SRCS + table_test.cc + DEPS + common_table + table + ps_framework_proto + ${RPC_DEPS}) set_source_files_properties( dense_table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( +paddle_test( dense_table_test - SRCS dense_table_test.cc - DEPS common_table table ps_framework_proto ${COMMON_DEPS} ${RPC_DEPS}) + SRCS + dense_table_test.cc + DEPS + common_table + table + ps_framework_proto + ${RPC_DEPS}) set_source_files_properties( barrier_table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( +paddle_test( barrier_table_test - SRCS barrier_table_test.cc - DEPS common_table table ps_framework_proto ${COMMON_DEPS}) + SRCS + barrier_table_test.cc + DEPS + common_table + table + ps_framework_proto) set_source_files_properties( brpc_service_dense_sgd_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( +paddle_test( brpc_service_dense_sgd_test - SRCS brpc_service_dense_sgd_test.cc - DEPS scope ps_service table ps_framework_proto ${COMMON_DEPS}) + SRCS + brpc_service_dense_sgd_test.cc + DEPS + scope + ps_service + table + ps_framework_proto) set_source_files_properties( brpc_service_sparse_sgd_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( +paddle_test( brpc_service_sparse_sgd_test - SRCS brpc_service_sparse_sgd_test.cc - DEPS scope ps_service table ps_framework_proto ${COMMON_DEPS}) + SRCS + brpc_service_sparse_sgd_test.cc + DEPS + scope + ps_service + table + ps_framework_proto) set_source_files_properties( brpc_utils_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( +paddle_test( brpc_utils_test - SRCS brpc_utils_test.cc - DEPS brpc_utils - scope - phi - common - sendrecv_rpc - ps_service - ${COMMON_DEPS} - ${RPC_DEPS}) + SRCS + brpc_utils_test.cc + DEPS + brpc_utils + scope + phi + common + sendrecv_rpc + ps_service + ${RPC_DEPS}) set_source_files_properties( graph_node_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( +paddle_test( graph_node_test - SRCS graph_node_test.cc - DEPS scope ps_service table ps_framework_proto ${COMMON_DEPS}) + SRCS + graph_node_test.cc + DEPS + scope + ps_service + table + ps_framework_proto) set_source_files_properties( graph_node_split_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( +paddle_test( graph_node_split_test - SRCS graph_node_split_test.cc - DEPS scope ps_service table ps_framework_proto ${COMMON_DEPS}) + SRCS + graph_node_split_test.cc + DEPS + scope + ps_service + table + ps_framework_proto) set_source_files_properties( graph_table_sample_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( - graph_table_sample_test - SRCS graph_table_sample_test.cc - DEPS table ps_framework_proto ${COMMON_DEPS}) +paddle_test(graph_table_sample_test SRCS graph_table_sample_test.cc DEPS table + ps_framework_proto) set_source_files_properties( feature_value_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( +paddle_test( feature_value_test - SRCS feature_value_test.cc - DEPS table common_table sendrecv_rpc ${COMMON_DEPS}) + SRCS + feature_value_test.cc + DEPS + table + common_table + sendrecv_rpc) set_source_files_properties( sparse_sgd_rule_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( - sparse_sgd_rule_test - SRCS sparse_sgd_rule_test.cc - DEPS ${COMMON_DEPS} table) +paddle_test(sparse_sgd_rule_test SRCS sparse_sgd_rule_test.cc DEPS table) set_source_files_properties( ctr_accessor_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( - ctr_accessor_test - SRCS ctr_accessor_test.cc - DEPS ${COMMON_DEPS} table) +paddle_test(ctr_accessor_test SRCS ctr_accessor_test.cc DEPS table) set_source_files_properties( ctr_dymf_accessor_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( - ctr_dymf_accessor_test - SRCS ctr_dymf_accessor_test.cc - DEPS ${COMMON_DEPS} table) +paddle_test(ctr_dymf_accessor_test SRCS ctr_dymf_accessor_test.cc DEPS table) set_source_files_properties( memory_sparse_table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( - memory_sparse_table_test - SRCS memory_sparse_table_test.cc - DEPS ${COMMON_DEPS} table) +paddle_test(memory_sparse_table_test SRCS memory_sparse_table_test.cc DEPS + table) set_source_files_properties( memory_geo_table_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_test( - memory_sparse_geo_table_test - SRCS memory_geo_table_test.cc - DEPS ${COMMON_DEPS} table) +paddle_test(memory_sparse_geo_table_test SRCS memory_geo_table_test.cc DEPS + table) diff --git a/paddle/fluid/distributed/test/barrier_table_test.cc b/test/cpp/fluid/distributed/barrier_table_test.cc similarity index 100% rename from paddle/fluid/distributed/test/barrier_table_test.cc rename to test/cpp/fluid/distributed/barrier_table_test.cc diff --git a/paddle/fluid/distributed/test/brpc_service_dense_sgd_test.cc b/test/cpp/fluid/distributed/brpc_service_dense_sgd_test.cc similarity index 100% rename from paddle/fluid/distributed/test/brpc_service_dense_sgd_test.cc rename to test/cpp/fluid/distributed/brpc_service_dense_sgd_test.cc diff --git a/paddle/fluid/distributed/test/brpc_service_sparse_sgd_test.cc b/test/cpp/fluid/distributed/brpc_service_sparse_sgd_test.cc similarity index 100% rename from paddle/fluid/distributed/test/brpc_service_sparse_sgd_test.cc rename to test/cpp/fluid/distributed/brpc_service_sparse_sgd_test.cc diff --git a/paddle/fluid/distributed/test/brpc_utils_test.cc b/test/cpp/fluid/distributed/brpc_utils_test.cc similarity index 100% rename from paddle/fluid/distributed/test/brpc_utils_test.cc rename to test/cpp/fluid/distributed/brpc_utils_test.cc diff --git a/paddle/fluid/distributed/test/ctr_accessor_test.cc b/test/cpp/fluid/distributed/ctr_accessor_test.cc similarity index 100% rename from paddle/fluid/distributed/test/ctr_accessor_test.cc rename to test/cpp/fluid/distributed/ctr_accessor_test.cc diff --git a/paddle/fluid/distributed/test/ctr_dymf_accessor_test.cc b/test/cpp/fluid/distributed/ctr_dymf_accessor_test.cc similarity index 100% rename from paddle/fluid/distributed/test/ctr_dymf_accessor_test.cc rename to test/cpp/fluid/distributed/ctr_dymf_accessor_test.cc diff --git a/paddle/fluid/distributed/test/dense_table_test.cc b/test/cpp/fluid/distributed/dense_table_test.cc similarity index 100% rename from paddle/fluid/distributed/test/dense_table_test.cc rename to test/cpp/fluid/distributed/dense_table_test.cc diff --git a/paddle/fluid/distributed/test/feature_value_test.cc b/test/cpp/fluid/distributed/feature_value_test.cc similarity index 100% rename from paddle/fluid/distributed/test/feature_value_test.cc rename to test/cpp/fluid/distributed/feature_value_test.cc diff --git a/paddle/fluid/distributed/test/graph_node_split_test.cc b/test/cpp/fluid/distributed/graph_node_split_test.cc similarity index 100% rename from paddle/fluid/distributed/test/graph_node_split_test.cc rename to test/cpp/fluid/distributed/graph_node_split_test.cc diff --git a/paddle/fluid/distributed/test/graph_node_test.cc b/test/cpp/fluid/distributed/graph_node_test.cc similarity index 100% rename from paddle/fluid/distributed/test/graph_node_test.cc rename to test/cpp/fluid/distributed/graph_node_test.cc diff --git a/paddle/fluid/distributed/test/graph_table_sample_test.cc b/test/cpp/fluid/distributed/graph_table_sample_test.cc similarity index 100% rename from paddle/fluid/distributed/test/graph_table_sample_test.cc rename to test/cpp/fluid/distributed/graph_table_sample_test.cc diff --git a/paddle/fluid/distributed/test/memory_geo_table_test.cc b/test/cpp/fluid/distributed/memory_geo_table_test.cc similarity index 100% rename from paddle/fluid/distributed/test/memory_geo_table_test.cc rename to test/cpp/fluid/distributed/memory_geo_table_test.cc diff --git a/paddle/fluid/distributed/test/memory_sparse_table_test.cc b/test/cpp/fluid/distributed/memory_sparse_table_test.cc similarity index 100% rename from paddle/fluid/distributed/test/memory_sparse_table_test.cc rename to test/cpp/fluid/distributed/memory_sparse_table_test.cc diff --git a/paddle/fluid/distributed/test/sparse_sgd_rule_test.cc b/test/cpp/fluid/distributed/sparse_sgd_rule_test.cc similarity index 100% rename from paddle/fluid/distributed/test/sparse_sgd_rule_test.cc rename to test/cpp/fluid/distributed/sparse_sgd_rule_test.cc diff --git a/paddle/fluid/distributed/test/table_test.cc b/test/cpp/fluid/distributed/table_test.cc similarity index 100% rename from paddle/fluid/distributed/test/table_test.cc rename to test/cpp/fluid/distributed/table_test.cc diff --git a/test/cpp/fluid/pscore/CMakeLists.txt b/test/cpp/fluid/pscore/CMakeLists.txt index c95841199d76b..3b74fd0a6f793 100644 --- a/test/cpp/fluid/pscore/CMakeLists.txt +++ b/test/cpp/fluid/pscore/CMakeLists.txt @@ -67,9 +67,6 @@ set_source_files_properties( ${DISTRIBUTE_COMPILE_FLAGS}) paddle_test(heter_listen_and_server_test SRCS heter_listen_and_server_test.cc) -#set_source_files_properties(heter_cloud_comm_cpu_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -#cc_test(heter_cloud_comm_cpu_test SRCS heter_cloud_comm_cpu_test.cc DEPS executor scope proto_desc generated_static_op heter_listen_and_serv_op ${RPC_DEPS} ${DISTRIBUTE_DEPS} phi common) - set_source_files_properties( switch_server_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) paddle_test(switch_server_test SRCS switch_server_test.cc)