Skip to content
This repository has been archived by the owner on Jun 23, 2022. It is now read-only.

fix: memory leak in nfs test #757

Merged
merged 1 commit into from
Feb 9, 2021
Merged

fix: memory leak in nfs test #757

merged 1 commit into from
Feb 9, 2021

Conversation

levy5307
Copy link
Contributor

@levy5307 levy5307 commented Feb 9, 2021

the error message that asan reported:

==24019==ERROR: LeakSanitizer: detected memory leaks

Direct leak of 448 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a0ce3adf in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:154
    #2 0x5649a0ce3adf in operator() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/bind_handler.hpp:65
    #3 0x5649a0ce3adf in asio_handler_invoke<boost::asio::detail::binder1<dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/handler_invoke_hook.hpp:69
    #4 0x5649a0ce3adf in invoke<boost::asio::detail::binder1<dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)>, boost::system::error_code>, dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_invoke_helpers.hpp:37
    #5 0x5649a0ce3adf in complete<boost::asio::detail::binder1<dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_work.hpp:82
    #6 0x5649a0ce3adf in do_complete /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/reactive_socket_accept_op.hpp:137
    #7 0x5649a0cf9aea in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #8 0x5649a0cf9aea in boost::asio::detail::epoll_reactor::descriptor_state::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/epoll_reactor.ipp:776
    #9 0x5649a0a59c12 in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #10 0x5649a0a59c12 in boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock&, boost::asio::detail::scheduler_thread_info&, boost::system::error_code const&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:401
    #11 0x5649a0a59c12 in boost::asio::detail::scheduler::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:154
    #12 0x5649a0cdaf3f in boost::asio::io_context::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/impl/io_context.ipp:70
    #13 0x5649a0cdaf3f in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:79
    #14 0x5649a0cdaf3f in __invoke_impl<void, dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:60
    #15 0x5649a0cdaf3f in __invoke<dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:95
    #16 0x5649a0cdaf3f in _M_invoke<0> /usr/include/c++/7/thread:234
    #17 0x5649a0cdaf3f in operator() /usr/include/c++/7/thread:243
    #18 0x5649a0cdaf3f in _M_run /usr/include/c++/7/thread:186
    #19 0x7efe0ec8f6de  (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0xbd6de)

Direct leak of 448 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a0cdf184 in dsn::tools::asio_network_provider::create_client_session(dsn::rpc_address) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:131
    #2 0x5649a0ac16fd in dsn::connection_oriented_network::send_message(dsn::message_ex*) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:638
    #3 0x5649a0ad4d05 in dsn::rpc_engine::call_ip(dsn::rpc_address, dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&, bool, bool) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:718
    #4 0x5649a0ad7d7b in dsn::rpc_engine::call_address(dsn::rpc_address, dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.h:202
    #5 0x5649a0ad7d7b in dsn::rpc_engine::call(dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:617
    #6 0x5649a098a359 in dsn_rpc_call(dsn::rpc_address, dsn::rpc_response_task*) /home/mi/workspace/pegasus/rdsn/src/runtime/service_api_c.cpp:128
    #7 0x5649a096ce93 in call<dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/async_calls.h:150
    #8 0x5649a096ce93 in call<const dsn::service::get_file_size_request&, dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/async_calls.h:179
    #9 0x5649a096ce93 in get_file_size<dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_client.h:123
    #10 0x5649a096ce93 in dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_client_impl.cpp:138
    #11 0x5649a08ede7e in dsn::nfs_node::copy_remote_files(dsn::rpc_address, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool, dsn::task_code, dsn::task_tracker*, std::function<void (dsn::error_code, unsigned long)>&&, int) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_node.cpp:56
    #12 0x5649a08cc512 in nfs_basic_Test::TestBody() /home/mi/workspace/pegasus/rdsn/src/nfs/test/main.cpp:52
    #13 0x5649a0db98ab in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7648ab)
    #14 0x5649a0db3b2e in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x75eb2e)
    #15 0x5649a0d9755b in testing::Test::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x74255b)
    #16 0x5649a0d97e83 in testing::TestInfo::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x742e83)
    #17 0x5649a0d984fb in testing::TestCase::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7434fb)
    #18 0x5649a0d9f39f in testing::internal::UnitTestImpl::RunAllTests() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x74a39f)
    #19 0x5649a0dba9d2 in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7659d2)
    #20 0x5649a0db496a in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x75f96a)
    #21 0x5649a0d9df85 in testing::UnitTest::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x748f85)
    #22 0x5649a0874e7e in RUN_ALL_TESTS() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/gtest/gtest.h:2233
    #23 0x5649a0874e7e in main /home/mi/workspace/pegasus/rdsn/src/nfs/test/main.cpp:149
    #24 0x7efe0e5eab96 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)

Indirect leak of 65536 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32608 in operator new[](unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0608)
    #1 0x5649a091a670 in std::shared_ptr<char> dsn::utils::make_shared_array<char>(unsigned long) /home/mi/workspace/pegasus/rdsn/include/dsn/utility/utils.h:72
    #2 0x5649a0ab050d in dsn::message_reader::read_buffer_ptr(unsigned int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/message_parser.cpp:150
    #3 0x5649a0cfd678 in dsn::tools::asio_rpc_session::do_read(int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_rpc_session.cpp:83
    #4 0x5649a0ab5dff in dsn::rpc_session::start_read_next(int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:224
    #5 0x5649a0d1309c in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_rpc_session.cpp:197
    #6 0x5649a0d1309c in operator() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/bind_handler.hpp:65
    #7 0x5649a0d1309c in asio_handler_invoke<boost::asio::detail::binder1<dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/handler_invoke_hook.hpp:69
    #8 0x5649a0d1309c in invoke<boost::asio::detail::binder1<dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)>, boost::system::error_code>, dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_invoke_helpers.hpp:37
    #9 0x5649a0d1309c in complete<boost::asio::detail::binder1<dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_work.hpp:82
    #10 0x5649a0d1309c in do_complete /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/reactive_socket_connect_op.hpp:100
    #11 0x5649a0cf9aea in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #12 0x5649a0cf9aea in boost::asio::detail::epoll_reactor::descriptor_state::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/epoll_reactor.ipp:776
    #13 0x5649a0a59c12 in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #14 0x5649a0a59c12 in boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock&, boost::asio::detail::scheduler_thread_info&, boost::system::error_code const&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:401
    #15 0x5649a0a59c12 in boost::asio::detail::scheduler::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:154
    #16 0x5649a0cdaf3f in boost::asio::io_context::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/impl/io_context.ipp:70
    #17 0x5649a0cdaf3f in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:79
    #18 0x5649a0cdaf3f in __invoke_impl<void, dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:60
    #19 0x5649a0cdaf3f in __invoke<dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:95
    #20 0x5649a0cdaf3f in _M_invoke<0> /usr/include/c++/7/thread:234
    #21 0x5649a0cdaf3f in operator() /usr/include/c++/7/thread:243
    #22 0x5649a0cdaf3f in _M_run /usr/include/c++/7/thread:186
    #23 0x7efe0ec8f6de  (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0xbd6de)

Indirect leak of 65536 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32608 in operator new[](unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0608)
    #1 0x5649a091a670 in std::shared_ptr<char> dsn::utils::make_shared_array<char>(unsigned long) /home/mi/workspace/pegasus/rdsn/include/dsn/utility/utils.h:72
    #2 0x5649a0ab050d in dsn::message_reader::read_buffer_ptr(unsigned int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/message_parser.cpp:150
    #3 0x5649a0cfd678 in dsn::tools::asio_rpc_session::do_read(int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_rpc_session.cpp:83
    #4 0x5649a0ab5dff in dsn::rpc_session::start_read_next(int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:224
    #5 0x5649a0ce43b0 in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:167
    #6 0x5649a0ce43b0 in operator() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/bind_handler.hpp:65
    #7 0x5649a0ce43b0 in asio_handler_invoke<boost::asio::detail::binder1<dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/handler_invoke_hook.hpp:69
    #8 0x5649a0ce43b0 in invoke<boost::asio::detail::binder1<dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)>, boost::system::error_code>, dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_invoke_helpers.hpp:37
    #9 0x5649a0ce43b0 in complete<boost::asio::detail::binder1<dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_work.hpp:82
    #10 0x5649a0ce43b0 in do_complete /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/reactive_socket_accept_op.hpp:137
    #11 0x5649a0cf9aea in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #12 0x5649a0cf9aea in boost::asio::detail::epoll_reactor::descriptor_state::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/epoll_reactor.ipp:776
    #13 0x5649a0a59c12 in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #14 0x5649a0a59c12 in boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock&, boost::asio::detail::scheduler_thread_info&, boost::system::error_code const&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:401
    #15 0x5649a0a59c12 in boost::asio::detail::scheduler::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:154
    #16 0x5649a0cdaf3f in boost::asio::io_context::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/impl/io_context.ipp:70
    #17 0x5649a0cdaf3f in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:79
    #18 0x5649a0cdaf3f in __invoke_impl<void, dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:60
    #19 0x5649a0cdaf3f in __invoke<dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:95
    #20 0x5649a0cdaf3f in _M_invoke<0> /usr/include/c++/7/thread:234
    #21 0x5649a0cdaf3f in operator() /usr/include/c++/7/thread:243
    #22 0x5649a0cdaf3f in _M_run /usr/include/c++/7/thread:186
    #23 0x7efe0ec8f6de  (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0xbd6de)

Indirect leak of 64 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a0ac60a6 in __gnu_cxx::new_allocator<dsn::message_parser::send_buf>::allocate(unsigned long, void const*) /usr/include/c++/7/ext/new_allocator.h:111
    #2 0x5649a0ac60a6 in std::allocator_traits<std::allocator<dsn::message_parser::send_buf> >::allocate(std::allocator<dsn::message_parser::send_buf>&, unsigned long) /usr/include/c++/7/bits/alloc_traits.h:436
    #3 0x5649a0ac60a6 in std::_Vector_base<dsn::message_parser::send_buf, std::allocator<dsn::message_parser::send_buf> >::_M_allocate(unsigned long) /usr/include/c++/7/bits/stl_vector.h:172
    #4 0x5649a0ac60a6 in std::vector<dsn::message_parser::send_buf, std::allocator<dsn::message_parser::send_buf> >::_M_default_append(unsigned long) /usr/include/c++/7/bits/vector.tcc:571
    #5 0x5649a0ac6fd6 in std::vector<dsn::message_parser::send_buf, std::allocator<dsn::message_parser::send_buf> >::resize(unsigned long) /usr/include/c++/7/bits/stl_vector.h:692
    #6 0x5649a0ac6fd6 in dsn::rpc_session::unlink_message_for_send() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:187
    #7 0x5649a0abc1df in dsn::rpc_session::send_message(dsn::message_ex*) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:275
    #8 0x5649a0ad207e in dsn::rpc_engine::reply(dsn::message_ex*, dsn::error_code) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:761
    #9 0x5649a0916625 in dsn::rpc_replier<dsn::service::copy_response>::operator()(dsn::service::copy_response const&) /home/mi/workspace/pegasus/rdsn/include/dsn/cpp/serverlet.h:77
    #10 0x5649a0916625 in dsn::service::nfs_service_impl::internal_read_callback(dsn::error_code, unsigned long, dsn::service::nfs_service_impl::callback_para&) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_server_impl.cpp:157
    #11 0x5649a0dca671 in std::function<void (dsn::error_code, unsigned long)>::operator()(dsn::error_code, unsigned long) const /usr/include/c++/7/bits/std_function.h:706
    #12 0x5649a0dca671 in dsn::aio_task::exec() /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/aio_task.h:99
    #13 0x5649a0b08f2c in dsn::task::exec_internal() /home/mi/workspace/pegasus/rdsn/src/runtime/task/task.cpp:176
    #14 0x5649a0b6d445 in dsn::task_worker::loop() /home/mi/workspace/pegasus/rdsn/src/runtime/task/task_worker.cpp:211
    #15 0x5649a0b6e31c in dsn::task_worker::run_internal() /home/mi/workspace/pegasus/rdsn/src/runtime/task/task_worker.cpp:191
    #16 0x5649a0b6e8b3 in void std::__invoke_impl<void, void (dsn::task_worker::*&)(), dsn::task_worker*&>(std::__invoke_memfun_deref, void (dsn::task_worker::*&)(), dsn::task_worker*&) /usr/include/c++/7/bits/invoke.h:73
    #17 0x5649a0b6e8b3 in std::__invoke_result<void (dsn::task_worker::*&)(), dsn::task_worker*&>::type std::__invoke<void (dsn::task_worker::*&)(), dsn::task_worker*&>(void (dsn::task_worker::*&)(), dsn::task_worker*&) /usr/include/c++/7/bits/invoke.h:95
    #18 0x5649a0b6e8b3 in void std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>::__call<void, , 0ul>(std::tuple<>&&, std::_Index_tuple<0ul>) /usr/include/c++/7/functional:467
    #19 0x5649a0b6e8b3 in void std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>::operator()<, void>() /usr/include/c++/7/functional:551
    #20 0x5649a0b6e8b3 in void std::__invoke_impl<void, std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>>(std::__invoke_other, std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>&&) /usr/include/c++/7/bits/invoke.h:60
    #21 0x5649a0b6e8b3 in std::__invoke_result<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>>::type std::__invoke<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>>(std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>&&) /usr/include/c++/7/bits/invoke.h:95
    #22 0x5649a0b6e8b3 in decltype (__invoke((_S_declval<0ul>)())) std::thread::_Invoker<std::tuple<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()> > >::_M_invoke<0ul>(std::_Index_tuple<0ul>) /usr/include/c++/7/thread:234
    #23 0x5649a0b6e8b3 in std::thread::_Invoker<std::tuple<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()> > >::operator()() /usr/include/c++/7/thread:243
    #24 0x5649a0b6e8b3 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()> > > >::_M_run() /usr/include/c++/7/thread:186
    #25 0x7efe0ec8f6de  (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0xbd6de)

Indirect leak of 48 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a0cdec4e in __gnu_cxx::new_allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> >::allocate(unsigned long, void const*) /usr/include/c++/7/ext/new_allocator.h:111
    #2 0x5649a0cdec4e in std::allocator_traits<std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> > >::allocate(std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> >&, unsigned long) /usr/include/c++/7/bits/alloc_traits.h:436
    #3 0x5649a0cdec4e in std::__allocated_ptr<std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> > > std::__allocate_guarded<std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> > >(std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> >&) /usr/include/c++/7/bits/allocated_ptr.h:104
    #4 0x5649a0cdec4e in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::_Sp_make_shared_tag, boost::asio::basic_stream_socket<boost::asio::ip::tcp>*, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr_base.h:635
    #5 0x5649a0cdec4e in std::__shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, (__gnu_cxx::_Lock_policy)2>::__shared_ptr<std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::_Sp_make_shared_tag, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr_base.h:1295
    #6 0x5649a0cdec4e in std::shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >::shared_ptr<std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::_Sp_make_shared_tag, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr.h:344
    #7 0x5649a0cdec4e in std::shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > std::allocate_shared<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr.h:691
    #8 0x5649a0cdec4e in std::shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > std::make_shared<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, boost::asio::io_context&>(boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr.h:707
    #9 0x5649a0cdec4e in dsn::tools::asio_network_provider::create_client_session(dsn::rpc_address) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:129
    #10 0x5649a0ac16fd in dsn::connection_oriented_network::send_message(dsn::message_ex*) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:638
    #11 0x5649a0ad4d05 in dsn::rpc_engine::call_ip(dsn::rpc_address, dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&, bool, bool) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:718
    #12 0x5649a0ad7d7b in dsn::rpc_engine::call_address(dsn::rpc_address, dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.h:202
    #13 0x5649a0ad7d7b in dsn::rpc_engine::call(dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:617
    #14 0x5649a098a359 in dsn_rpc_call(dsn::rpc_address, dsn::rpc_response_task*) /home/mi/workspace/pegasus/rdsn/src/runtime/service_api_c.cpp:128
    #15 0x5649a096ce93 in call<dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/async_calls.h:150
    #16 0x5649a096ce93 in call<const dsn::service::get_file_size_request&, dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/async_calls.h:179
    #17 0x5649a096ce93 in get_file_size<dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_client.h:123
    #18 0x5649a096ce93 in dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_client_impl.cpp:138
    #19 0x5649a08ede7e in dsn::nfs_node::copy_remote_files(dsn::rpc_address, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool, dsn::task_code, dsn::task_tracker*, std::function<void (dsn::error_code, unsigned long)>&&, int) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_node.cpp:56
    #20 0x5649a08cc512 in nfs_basic_Test::TestBody() /home/mi/workspace/pegasus/rdsn/src/nfs/test/main.cpp:52
    #21 0x5649a0db98ab in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7648ab)
    #22 0x5649a0db3b2e in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x75eb2e)
    #23 0x5649a0d9755b in testing::Test::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x74255b)
    #24 0x5649a0d97e83 in testing::TestInfo::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x742e83)
    #25 0x5649a0d984fb in testing::TestCase::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7434fb)
    #26 0x5649a0d9f39f in testing::internal::UnitTestImpl::RunAllTests() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x74a39f)
    #27 0x5649a0dba9d2 in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7659d2)
 #28 0x5649a0db496a in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x75f96a)
    #29 0x5649a0d9df85 in testing::UnitTest::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x748f85)
    #30 0x5649a0874e7e in RUN_ALL_TESTS() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/gtest/gtest.h:2233
    #31 0x5649a0874e7e in main /home/mi/workspace/pegasus/rdsn/src/nfs/test/main.cpp:149
    #32 0x7efe0e5eab96 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)

Indirect leak of 48 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a0cdf854 in __gnu_cxx::new_allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> >::allocate(unsigned long, void const*) /usr/include/c++/7/ext/new_allocator.h:111
    #2 0x5649a0cdf854 in std::allocator_traits<std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> > >::allocate(std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> >&, unsigned long) /usr/include/c++/7/bits/alloc_traits.h:436
    #3 0x5649a0cdf854 in std::__allocated_ptr<std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> > > std::__allocate_guarded<std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> > >(std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> >&) /usr/include/c++/7/bits/allocated_ptr.h:104
    #4 0x5649a0cdf854 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::_Sp_make_shared_tag, boost::asio::basic_stream_socket<boost::asio::ip::tcp>*, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr_base.h:635
    #5 0x5649a0cdf854 in std::__shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, (__gnu_cxx::_Lock_policy)2>::__shared_ptr<std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::_Sp_make_shared_tag, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr_base.h:1295
    #6 0x5649a0cdf854 in std::shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >::shared_ptr<std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::_Sp_make_shared_tag, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr.h:344
    #7 0x5649a0cdf854 in std::shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > std::allocate_shared<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr.h:691
    #8 0x5649a0cdf854 in std::shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > std::make_shared<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, boost::asio::io_context&>(boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr.h:707
    #9 0x5649a0cdf854 in dsn::tools::asio_network_provider::do_accept() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:136
    #10 0x5649a0ce7a50 in dsn::tools::asio_network_provider::start(dsn::utils::customized_id<dsn::rpc_channel_>, int, bool) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:121
    #11 0x5649a0ad8964 in dsn::rpc_engine::create_network(dsn::network_server_config const&, bool, dsn::utils::customized_id<dsn::network_header_format_>) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:426
    #12 0x5649a0adad3d in dsn::rpc_engine::start(dsn::service_app_spec const&) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:499
    #13 0x5649a09bb4f9 in dsn::service_node::init_rpc_engine() /home/mi/workspace/pegasus/rdsn/src/runtime/service_engine.cpp:62
    #14 0x5649a09bba84 in dsn::service_node::start() /home/mi/workspace/pegasus/rdsn/src/runtime/service_engine.cpp:116
    #15 0x5649a09bd3e4 in dsn::service_engine::start_node(dsn::service_app_spec&) /home/mi/workspace/pegasus/rdsn/src/runtime/service_engine.cpp:238
    #16 0x5649a099dbfa in run /home/mi/workspace/pegasus/rdsn/src/runtime/service_api_c.cpp:499
    #17 0x5649a09a2a13 in dsn_run_config(char const*, bool) /home/mi/workspace/pegasus/rdsn/src/runtime/service_api_c.cpp:183
    #18 0x5649a0874e71 in main /home/mi/workspace/pegasus/rdsn/src/nfs/test/main.cpp:148
    #19 0x7efe0e5eab96 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)

Indirect leak of 32 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a0c9319d in dsn::message_parser* dsn::message_parser::create<dsn::dsn_message_parser>() /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/message_parser.h:90
    #2 0x5649a0abe0ba in dsn::network::new_message_parser(dsn::utils::customized_id<dsn::network_header_format_>) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:547
    #3 0x5649a0cdf0db in dsn::tools::asio_network_provider::create_client_session(dsn::rpc_address) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:130
    #4 0x5649a0ac16fd in dsn::connection_oriented_network::send_message(dsn::message_ex*) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:638
    #5 0x5649a0ad4d05 in dsn::rpc_engine::call_ip(dsn::rpc_address, dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&, bool, bool) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:718
    #6 0x5649a0ad7d7b in dsn::rpc_engine::call_address(dsn::rpc_address, dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.h:202
    #7 0x5649a0ad7d7b in dsn::rpc_engine::call(dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:617
    #8 0x5649a098a359 in dsn_rpc_call(dsn::rpc_address, dsn::rpc_response_task*) /home/mi/workspace/pegasus/rdsn/src/runtime/service_api_c.cpp:128
    #9 0x5649a096ce93 in call<dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/async_calls.h:150
    #10 0x5649a096ce93 in call<const dsn::service::get_file_size_request&, dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/async_calls.h:179
    #11 0x5649a096ce93 in get_file_size<dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_client.h:123
    #12 0x5649a096ce93 in dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_client_impl.cpp:138
    #13 0x5649a08ede7e in dsn::nfs_node::copy_remote_files(dsn::rpc_address, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool, dsn::task_code, dsn::task_tracker*, std::function<void (dsn::error_code, unsigned long)>&&, int) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_node.cpp:56
    #14 0x5649a08cc512 in nfs_basic_Test::TestBody() /home/mi/workspace/pegasus/rdsn/src/nfs/test/main.cpp:52
    #15 0x5649a0db98ab in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7648ab)
    #16 0x5649a0db3b2e in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x75eb2e)
    #17 0x5649a0d9755b in testing::Test::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x74255b)
    #18 0x5649a0d97e83 in testing::TestInfo::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x742e83)
    #19 0x5649a0d984fb in testing::TestCase::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7434fb)
    #20 0x5649a0d9f39f in testing::internal::UnitTestImpl::RunAllTests() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x74a39f)
    #21 0x5649a0dba9d2 in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7659d2)
    #22 0x5649a0db496a in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x75f96a)
    #23 0x5649a0d9df85 in testing::UnitTest::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x748f85)
    #24 0x5649a0874e7e in RUN_ALL_TESTS() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/gtest/gtest.h:2233
    #25 0x5649a0874e7e in main /home/mi/workspace/pegasus/rdsn/src/nfs/test/main.cpp:149
    #26 0x7efe0e5eab96 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)

Indirect leak of 65536 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32608 in operator new[](unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0608)
    #1 0x5649a091a670 in std::shared_ptr<char> dsn::utils::make_shared_array<char>(unsigned long) /home/mi/workspace/pegasus/rdsn/include/dsn/utility/utils.h:72
    #2 0x5649a0ab050d in dsn::message_reader::read_buffer_ptr(unsigned int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/message_parser.cpp:150
    #3 0x5649a0cfd678 in dsn::tools::asio_rpc_session::do_read(int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_rpc_session.cpp:83
    #4 0x5649a0ab5dff in dsn::rpc_session::start_read_next(int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:224
    #5 0x5649a0d1309c in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_rpc_session.cpp:197
    #6 0x5649a0d1309c in operator() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/bind_handler.hpp:65
    #7 0x5649a0d1309c in asio_handler_invoke<boost::asio::detail::binder1<dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/handler_invoke_hook.hpp:69
    #8 0x5649a0d1309c in invoke<boost::asio::detail::binder1<dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)>, boost::system::error_code>, dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_invoke_helpers.hpp:37
    #9 0x5649a0d1309c in complete<boost::asio::detail::binder1<dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_work.hpp:82
    #10 0x5649a0d1309c in do_complete /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/reactive_socket_connect_op.hpp:100
    #11 0x5649a0cf9aea in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #12 0x5649a0cf9aea in boost::asio::detail::epoll_reactor::descriptor_state::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/epoll_reactor.ipp:776
    #13 0x5649a0a59c12 in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #14 0x5649a0a59c12 in boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock&, boost::asio::detail::scheduler_thread_info&, boost::system::error_code const&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:401
    #15 0x5649a0a59c12 in boost::asio::detail::scheduler::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:154
    #16 0x5649a0cdaf3f in boost::asio::io_context::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/impl/io_context.ipp:70
    #17 0x5649a0cdaf3f in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:79
    #18 0x5649a0cdaf3f in __invoke_impl<void, dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:60
    #19 0x5649a0cdaf3f in __invoke<dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:95
    #20 0x5649a0cdaf3f in _M_invoke<0> /usr/include/c++/7/thread:234
    #21 0x5649a0cdaf3f in operator() /usr/include/c++/7/thread:243
    #22 0x5649a0cdaf3f in _M_run /usr/include/c++/7/thread:186
    #23 0x7efe0ec8f6de  (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0xbd6de)

Indirect leak of 65536 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32608 in operator new[](unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0608)
    #1 0x5649a091a670 in std::shared_ptr<char> dsn::utils::make_shared_array<char>(unsigned long) /home/mi/workspace/pegasus/rdsn/include/dsn/utility/utils.h:72
    #2 0x5649a0ab050d in dsn::message_reader::read_buffer_ptr(unsigned int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/message_parser.cpp:150
    #3 0x5649a0cfd678 in dsn::tools::asio_rpc_session::do_read(int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_rpc_session.cpp:83
    #4 0x5649a0ab5dff in dsn::rpc_session::start_read_next(int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:224
    #5 0x5649a0ce43b0 in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:167
    #6 0x5649a0ce43b0 in operator() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/bind_handler.hpp:65
    #7 0x5649a0ce43b0 in asio_handler_invoke<boost::asio::detail::binder1<dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/handler_invoke_hook.hpp:69
    #8 0x5649a0ce43b0 in invoke<boost::asio::detail::binder1<dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)>, boost::system::error_code>, dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_invoke_helpers.hpp:37
    #9 0x5649a0ce43b0 in complete<boost::asio::detail::binder1<dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_work.hpp:82
    #10 0x5649a0ce43b0 in do_complete /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/reactive_socket_accept_op.hpp:137
    #11 0x5649a0cf9aea in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #12 0x5649a0cf9aea in boost::asio::detail::epoll_reactor::descriptor_state::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/epoll_reactor.ipp:776
    #13 0x5649a0a59c12 in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #14 0x5649a0a59c12 in boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock&, boost::asio::detail::scheduler_thread_info&, boost::system::error_code const&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:401
    #15 0x5649a0a59c12 in boost::asio::detail::scheduler::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:154
    #16 0x5649a0cdaf3f in boost::asio::io_context::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/impl/io_context.ipp:70
    #17 0x5649a0cdaf3f in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:79
    #18 0x5649a0cdaf3f in __invoke_impl<void, dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:60
    #19 0x5649a0cdaf3f in __invoke<dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:95
    #20 0x5649a0cdaf3f in _M_invoke<0> /usr/include/c++/7/thread:234
    #21 0x5649a0cdaf3f in operator() /usr/include/c++/7/thread:243
    #22 0x5649a0cdaf3f in _M_run /usr/include/c++/7/thread:186
    #23 0x7efe0ec8f6de  (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0xbd6de)

Indirect leak of 64 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a0ac60a6 in __gnu_cxx::new_allocator<dsn::message_parser::send_buf>::allocate(unsigned long, void const*) /usr/include/c++/7/ext/new_allocator.h:111
    #2 0x5649a0ac60a6 in std::allocator_traits<std::allocator<dsn::message_parser::send_buf> >::allocate(std::allocator<dsn::message_parser::send_buf>&, unsigned long) /usr/include/c++/7/bits/alloc_traits.h:436
    #3 0x5649a0ac60a6 in std::_Vector_base<dsn::message_parser::send_buf, std::allocator<dsn::message_parser::send_buf> >::_M_allocate(unsigned long) /usr/include/c++/7/bits/stl_vector.h:172
    #4 0x5649a0ac60a6 in std::vector<dsn::message_parser::send_buf, std::allocator<dsn::message_parser::send_buf> >::_M_default_append(unsigned long) /usr/include/c++/7/bits/vector.tcc:571
    #5 0x5649a0ac6fd6 in std::vector<dsn::message_parser::send_buf, std::allocator<dsn::message_parser::send_buf> >::resize(unsigned long) /usr/include/c++/7/bits/stl_vector.h:692
    #6 0x5649a0ac6fd6 in dsn::rpc_session::unlink_message_for_send() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:187
    #7 0x5649a0abc1df in dsn::rpc_session::send_message(dsn::message_ex*) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:275
    #8 0x5649a0ad207e in dsn::rpc_engine::reply(dsn::message_ex*, dsn::error_code) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:761
    #9 0x5649a0916625 in dsn::rpc_replier<dsn::service::copy_response>::operator()(dsn::service::copy_response const&) /home/mi/workspace/pegasus/rdsn/include/dsn/cpp/serverlet.h:77
    #10 0x5649a0916625 in dsn::service::nfs_service_impl::internal_read_callback(dsn::error_code, unsigned long, dsn::service::nfs_service_impl::callback_para&) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_server_impl.cpp:157
    #11 0x5649a0dca671 in std::function<void (dsn::error_code, unsigned long)>::operator()(dsn::error_code, unsigned long) const /usr/include/c++/7/bits/std_function.h:706
    #12 0x5649a0dca671 in dsn::aio_task::exec() /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/aio_task.h:99
    #13 0x5649a0b08f2c in dsn::task::exec_internal() /home/mi/workspace/pegasus/rdsn/src/runtime/task/task.cpp:176
    #14 0x5649a0b6d445 in dsn::task_worker::loop() /home/mi/workspace/pegasus/rdsn/src/runtime/task/task_worker.cpp:211
    #15 0x5649a0b6e31c in dsn::task_worker::run_internal() /home/mi/workspace/pegasus/rdsn/src/runtime/task/task_worker.cpp:191
    #16 0x5649a0b6e8b3 in void std::__invoke_impl<void, void (dsn::task_worker::*&)(), dsn::task_worker*&>(std::__invoke_memfun_deref, void (dsn::task_worker::*&)(), dsn::task_worker*&) /usr/include/c++/7/bits/invoke.h:73
    #17 0x5649a0b6e8b3 in std::__invoke_result<void (dsn::task_worker::*&)(), dsn::task_worker*&>::type std::__invoke<void (dsn::task_worker::*&)(), dsn::task_worker*&>(void (dsn::task_worker::*&)(), dsn::task_worker*&) /usr/include/c++/7/bits/invoke.h:95
    #18 0x5649a0b6e8b3 in void std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>::__call<void, , 0ul>(std::tuple<>&&, std::_Index_tuple<0ul>) /usr/include/c++/7/functional:467
    #19 0x5649a0b6e8b3 in void std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>::operator()<, void>() /usr/include/c++/7/functional:551
    #20 0x5649a0b6e8b3 in void std::__invoke_impl<void, std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>>(std::__invoke_other, std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>&&) /usr/include/c++/7/bits/invoke.h:60
    #21 0x5649a0b6e8b3 in std::__invoke_result<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>>::type std::__invoke<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>>(std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>&&) /usr/include/c++/7/bits/invoke.h:95
    #22 0x5649a0b6e8b3 in decltype (__invoke((_S_declval<0ul>)())) std::thread::_Invoker<std::tuple<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()> > >::_M_invoke<0ul>(std::_Index_tuple<0ul>) /usr/include/c++/7/thread:234
    #23 0x5649a0b6e8b3 in std::thread::_Invoker<std::tuple<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()> > >::operator()() /usr/include/c++/7/thread:243
    #24 0x5649a0b6e8b3 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()> > > >::_M_run() /usr/include/c++/7/thread:186
    #25 0x7efe0ec8f6de  (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0xbd6de)

Indirect leak of 48 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a0cdec4e in __gnu_cxx::new_allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> >::allocate(unsigned long, void const*) /usr/include/c++/7/ext/new_allocator.h:111
    #2 0x5649a0cdec4e in std::allocator_traits<std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> > >::allocate(std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> >&, unsigned long) /usr/include/c++/7/bits/alloc_traits.h:436
    #3 0x5649a0cdec4e in std::__allocated_ptr<std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> > > std::__allocate_guarded<std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> > >(std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> >&) /usr/include/c++/7/bits/allocated_ptr.h:104
    #4 0x5649a0cdec4e in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::_Sp_make_shared_tag, boost::asio::basic_stream_socket<boost::asio::ip::tcp>*, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr_base.h:635
    #5 0x5649a0cdec4e in std::__shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, (__gnu_cxx::_Lock_policy)2>::__shared_ptr<std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::_Sp_make_shared_tag, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr_base.h:1295
    #6 0x5649a0cdec4e in std::shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >::shared_ptr<std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::_Sp_make_shared_tag, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr.h:344
    #7 0x5649a0cdec4e in std::shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > std::allocate_shared<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr.h:691
    #8 0x5649a0cdec4e in std::shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > std::make_shared<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, boost::asio::io_context&>(boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr.h:707
    #9 0x5649a0cdec4e in dsn::tools::asio_network_provider::create_client_session(dsn::rpc_address) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:129
    #10 0x5649a0ac16fd in dsn::connection_oriented_network::send_message(dsn::message_ex*) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:638
    #11 0x5649a0ad4d05 in dsn::rpc_engine::call_ip(dsn::rpc_address, dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&, bool, bool) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:718
    #12 0x5649a0ad7d7b in dsn::rpc_engine::call_address(dsn::rpc_address, dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.h:202
    #13 0x5649a0ad7d7b in dsn::rpc_engine::call(dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:617
    #14 0x5649a098a359 in dsn_rpc_call(dsn::rpc_address, dsn::rpc_response_task*) /home/mi/workspace/pegasus/rdsn/src/runtime/service_api_c.cpp:128
    #15 0x5649a096ce93 in call<dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/async_calls.h:150
    #16 0x5649a096ce93 in call<const dsn::service::get_file_size_request&, dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/async_calls.h:179
    #17 0x5649a096ce93 in get_file_size<dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_client.h:123
    #18 0x5649a096ce93 in dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_client_impl.cpp:138
    #19 0x5649a08ede7e in dsn::nfs_node::copy_remote_files(dsn::rpc_address, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool, dsn::task_code, dsn::task_tracker*, std::function<void (dsn::error_code, unsigned long)>&&, int) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_node.cpp:56
    #20 0x5649a08cc512 in nfs_basic_Test::TestBody() /home/mi/workspace/pegasus/rdsn/src/nfs/test/main.cpp:52
    #21 0x5649a0db98ab in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7648ab)
    #22 0x5649a0db3b2e in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x75eb2e)
    #23 0x5649a0d9755b in testing::Test::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x74255b)
    #24 0x5649a0d97e83 in testing::TestInfo::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x742e83)
    #25 0x5649a0d984fb in testing::TestCase::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7434fb)
    #26 0x5649a0d9f39f in testing::internal::UnitTestImpl::RunAllTests() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x74a39f)
    #27 0x5649a0dba9d2 in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7659d2)
    #28 0x5649a0db496a in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x75f96a)
    #29 0x5649a0d9df85 in testing::UnitTest::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x748f85)
    #30 0x5649a0874e7e in RUN_ALL_TESTS() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/gtest/gtest.h:2233
    #31 0x5649a0874e7e in main /home/mi/workspace/pegasus/rdsn/src/nfs/test/main.cpp:149
    #32 0x7efe0e5eab96 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)

Indirect leak of 48 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a0cdf854 in __gnu_cxx::new_allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> >::allocate(unsigned long, void const*) /usr/include/c++/7/ext/new_allocator.h:111
    #2 0x5649a0cdf854 in std::allocator_traits<std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> > >::allocate(std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> >&, unsigned long) /usr/include/c++/7/bits/alloc_traits.h:436
    #3 0x5649a0cdf854 in std::__allocated_ptr<std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> > > std::__allocate_guarded<std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> > >(std::allocator<std::_Sp_counted_ptr_inplace<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, (__gnu_cxx::_Lock_policy)2> >&) /usr/include/c++/7/bits/allocated_ptr.h:104
    #4 0x5649a0cdf854 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::_Sp_make_shared_tag, boost::asio::basic_stream_socket<boost::asio::ip::tcp>*, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr_base.h:635
    #5 0x5649a0cdf854 in std::__shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, (__gnu_cxx::_Lock_policy)2>::__shared_ptr<std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::_Sp_make_shared_tag, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr_base.h:1295
    #6 0x5649a0cdf854 in std::shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >::shared_ptr<std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::_Sp_make_shared_tag, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr.h:344
    #7 0x5649a0cdf854 in std::shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > std::allocate_shared<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> >, boost::asio::io_context&>(std::allocator<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > const&, boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr.h:691
    #8 0x5649a0cdf854 in std::shared_ptr<boost::asio::basic_stream_socket<boost::asio::ip::tcp> > std::make_shared<boost::asio::basic_stream_socket<boost::asio::ip::tcp>, boost::asio::io_context&>(boost::asio::io_context&) /usr/include/c++/7/bits/shared_ptr.h:707
    #9 0x5649a0cdf854 in dsn::tools::asio_network_provider::do_accept() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:136
    #10 0x5649a0ce7a50 in dsn::tools::asio_network_provider::start(dsn::utils::customized_id<dsn::rpc_channel_>, int, bool) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:121
    #11 0x5649a0ad8964 in dsn::rpc_engine::create_network(dsn::network_server_config const&, bool, dsn::utils::customized_id<dsn::network_header_format_>) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:426
    #12 0x5649a0adad3d in dsn::rpc_engine::start(dsn::service_app_spec const&) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:499
    #13 0x5649a09bb4f9 in dsn::service_node::init_rpc_engine() /home/mi/workspace/pegasus/rdsn/src/runtime/service_engine.cpp:62
    #14 0x5649a09bba84 in dsn::service_node::start() /home/mi/workspace/pegasus/rdsn/src/runtime/service_engine.cpp:116
    #15 0x5649a09bd3e4 in dsn::service_engine::start_node(dsn::service_app_spec&) /home/mi/workspace/pegasus/rdsn/src/runtime/service_engine.cpp:238
    #16 0x5649a099dbfa in run /home/mi/workspace/pegasus/rdsn/src/runtime/service_api_c.cpp:499
    #17 0x5649a09a2a13 in dsn_run_config(char const*, bool) /home/mi/workspace/pegasus/rdsn/src/runtime/service_api_c.cpp:183
    #18 0x5649a0874e71 in main /home/mi/workspace/pegasus/rdsn/src/nfs/test/main.cpp:148
    #19 0x7efe0e5eab96 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)

Indirect leak of 32 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a0c9319d in dsn::message_parser* dsn::message_parser::create<dsn::dsn_message_parser>() /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/message_parser.h:90
    #2 0x5649a0abe0ba in dsn::network::new_message_parser(dsn::utils::customized_id<dsn::network_header_format_>) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:547
    #3 0x5649a0cdf0db in dsn::tools::asio_network_provider::create_client_session(dsn::rpc_address) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:130
    #4 0x5649a0ac16fd in dsn::connection_oriented_network::send_message(dsn::message_ex*) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:638
    #5 0x5649a0ad4d05 in dsn::rpc_engine::call_ip(dsn::rpc_address, dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&, bool, bool) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:718
    #6 0x5649a0ad7d7b in dsn::rpc_engine::call_address(dsn::rpc_address, dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.h:202
    #7 0x5649a0ad7d7b in dsn::rpc_engine::call(dsn::message_ex*, dsn::ref_ptr<dsn::rpc_response_task> const&) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:617
    #8 0x5649a098a359 in dsn_rpc_call(dsn::rpc_address, dsn::rpc_response_task*) /home/mi/workspace/pegasus/rdsn/src/runtime/service_api_c.cpp:128
    #9 0x5649a096ce93 in call<dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/async_calls.h:150
    #10 0x5649a096ce93 in call<const dsn::service::get_file_size_request&, dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/async_calls.h:179
    #11 0x5649a096ce93 in get_file_size<dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*)::<lambda(dsn::error_code, dsn::service::get_file_size_response&&)> > /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_client.h:123
    #12 0x5649a096ce93 in dsn::service::nfs_client_impl::begin_remote_copy(std::shared_ptr<dsn::remote_copy_request>&, dsn::aio_task*) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_client_impl.cpp:138
    #13 0x5649a08ede7e in dsn::nfs_node::copy_remote_files(dsn::rpc_address, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool, dsn::task_code, dsn::task_tracker*, std::function<void (dsn::error_code, unsigned long)>&&, int) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_node.cpp:56
    #14 0x5649a08cc512 in nfs_basic_Test::TestBody() /home/mi/workspace/pegasus/rdsn/src/nfs/test/main.cpp:52
    #15 0x5649a0db98ab in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7648ab)
    #16 0x5649a0db3b2e in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x75eb2e)
    #17 0x5649a0d9755b in testing::Test::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x74255b)
    #18 0x5649a0d97e83 in testing::TestInfo::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x742e83)
    #19 0x5649a0d984fb in testing::TestCase::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7434fb)
    #20 0x5649a0d9f39f in testing::internal::UnitTestImpl::RunAllTests() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x74a39f)
    #21 0x5649a0dba9d2 in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x7659d2)
    #22 0x5649a0db496a in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x75f96a)
    #23 0x5649a0d9df85 in testing::UnitTest::Run() (/home/mi/workspace/pegasus/rdsn/builder/src/nfs/test/dsn_nfs_test+0x748f85)
    #24 0x5649a0874e7e in RUN_ALL_TESTS() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/gtest/gtest.h:2233
    #25 0x5649a0874e7e in main /home/mi/workspace/pegasus/rdsn/src/nfs/test/main.cpp:149
    #26 0x7efe0e5eab96 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)

Indirect leak of 24 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a091a6b8 in __gnu_cxx::new_allocator<std::_Sp_counted_deleter<char*, std::default_delete<char []>, std::allocator<void>, (__gnu_cxx::_Lock_policy)2> >::allocate(unsigned long, void const*) /usr/include/c++/7/ext/new_allocator.h:111
    #2 0x5649a091a6b8 in std::allocator_traits<std::allocator<std::_Sp_counted_deleter<char*, std::default_delete<char []>, std::allocator<void>, (__gnu_cxx::_Lock_policy)2> > >::allocate(std::allocator<std::_Sp_counted_deleter<char*, std::default_delete<char []>, std::allocator<void>, (__gnu_cxx::_Lock_policy)2> >&, unsigned long) /usr/include/c++/7/bits/alloc_traits.h:436
    #3 0x5649a091a6b8 in std::__allocated_ptr<std::allocator<std::_Sp_counted_deleter<char*, std::default_delete<char []>, std::allocator<void>, (__gnu_cxx::_Lock_policy)2> > > std::__allocate_guarded<std::allocator<std::_Sp_counted_deleter<char*, std::default_delete<char []>, std::allocator<void>, (__gnu_cxx::_Lock_policy)2> > >(std::allocator<std::_Sp_counted_deleter<char*, std::default_delete<char []>, std::allocator<void>, (__gnu_cxx::_Lock_policy)2> >&) /usr/include/c++/7/bits/allocated_ptr.h:104
    #4 0x5649a091a6b8 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<char*, std::default_delete<char []>, std::allocator<void> >(char*, std::default_delete<char []>, std::allocator<void>) /usr/include/c++/7/bits/shared_ptr_base.h:615
    #5 0x5649a091a6b8 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<char*, std::default_delete<char []> >(char*, std::default_delete<char []>) /usr/include/c++/7/bits/shared_ptr_base.h:605
    #6 0x5649a091a6b8 in std::__shared_ptr<char, (__gnu_cxx::_Lock_policy)2>::__shared_ptr<char, std::default_delete<char []>, void>(char*, std::default_delete<char []>) /usr/include/c++/7/bits/shared_ptr_base.h:1088
    #7 0x5649a091a6b8 in std::shared_ptr<char>::shared_ptr<char, std::default_delete<char []>, void>(char*, std::default_delete<char []>) /usr/include/c++/7/bits/shared_ptr.h:147
    #8 0x5649a091a6b8 in std::shared_ptr<char> dsn::utils::make_shared_array<char>(unsigned long) /home/mi/workspace/pegasus/rdsn/include/dsn/utility/utils.h:72
    #9 0x5649a0ab050d in dsn::message_reader::read_buffer_ptr(unsigned int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/message_parser.cpp:150
    #10 0x5649a0cfd678 in dsn::tools::asio_rpc_session::do_read(int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_rpc_session.cpp:83
    #11 0x5649a0ab5dff in dsn::rpc_session::start_read_next(int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:224
    #12 0x5649a0ce43b0 in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:167
    #13 0x5649a0ce43b0 in operator() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/bind_handler.hpp:65
    #14 0x5649a0ce43b0 in asio_handler_invoke<boost::asio::detail::binder1<dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/handler_invoke_hook.hpp:69
    #15 0x5649a0ce43b0 in invoke<boost::asio::detail::binder1<dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)>, boost::system::error_code>, dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_invoke_helpers.hpp:37
    #16 0x5649a0ce43b0 in complete<boost::asio::detail::binder1<dsn::tools::asio_network_provider::do_accept()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_work.hpp:82
    #17 0x5649a0ce43b0 in do_complete /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/reactive_socket_accept_op.hpp:137
    #18 0x5649a0cf9aea in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #19 0x5649a0cf9aea in boost::asio::detail::epoll_reactor::descriptor_state::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/epoll_reactor.ipp:776
    #20 0x5649a0a59c12 in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #21 0x5649a0a59c12 in boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock&, boost::asio::detail::scheduler_thread_info&, boost::system::error_code const&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:401
    #22 0x5649a0a59c12 in boost::asio::detail::scheduler::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:154
    #23 0x5649a0cdaf3f in boost::asio::io_context::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/impl/io_context.ipp:70
    #24 0x5649a0cdaf3f in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:79
    #25 0x5649a0cdaf3f in __invoke_impl<void, dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:60
    #26 0x5649a0cdaf3f in __invoke<dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:95
    #27 0x5649a0cdaf3f in _M_invoke<0> /usr/include/c++/7/thread:234
    #28 0x5649a0cdaf3f in operator() /usr/include/c++/7/thread:243
    #29 0x5649a0cdaf3f in _M_run /usr/include/c++/7/thread:186
    #30 0x7efe0ec8f6de  (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0xbd6de)

Indirect leak of 24 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a091a6b8 in __gnu_cxx::new_allocator<std::_Sp_counted_deleter<char*, std::default_delete<char []>, std::allocator<void>, (__gnu_cxx::_Lock_policy)2> >::allocate(unsigned long, void const*) /usr/include/c++/7/ext/new_allocator.h:111
    #2 0x5649a091a6b8 in std::allocator_traits<std::allocator<std::_Sp_counted_deleter<char*, std::default_delete<char []>, std::allocator<void>, (__gnu_cxx::_Lock_policy)2> > >::allocate(std::allocator<std::_Sp_counted_deleter<char*, std::default_delete<char []>, std::allocator<void>, (__gnu_cxx::_Lock_policy)2> >&, unsigned long) /usr/include/c++/7/bits/alloc_traits.h:436
    #3 0x5649a091a6b8 in std::__allocated_ptr<std::allocator<std::_Sp_counted_deleter<char*, std::default_delete<char []>, std::allocator<void>, (__gnu_cxx::_Lock_policy)2> > > std::__allocate_guarded<std::allocator<std::_Sp_counted_deleter<char*, std::default_delete<char []>, std::allocator<void>, (__gnu_cxx::_Lock_policy)2> > >(std::allocator<std::_Sp_counted_deleter<char*, std::default_delete<char []>, std::allocator<void>, (__gnu_cxx::_Lock_policy)2> >&) /usr/include/c++/7/bits/allocated_ptr.h:104
    #4 0x5649a091a6b8 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<char*, std::default_delete<char []>, std::allocator<void> >(char*, std::default_delete<char []>, std::allocator<void>) /usr/include/c++/7/bits/shared_ptr_base.h:615
    #5 0x5649a091a6b8 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<char*, std::default_delete<char []> >(char*, std::default_delete<char []>) /usr/include/c++/7/bits/shared_ptr_base.h:605
    #6 0x5649a091a6b8 in std::__shared_ptr<char, (__gnu_cxx::_Lock_policy)2>::__shared_ptr<char, std::default_delete<char []>, void>(char*, std::default_delete<char []>) /usr/include/c++/7/bits/shared_ptr_base.h:1088
    #7 0x5649a091a6b8 in std::shared_ptr<char>::shared_ptr<char, std::default_delete<char []>, void>(char*, std::default_delete<char []>) /usr/include/c++/7/bits/shared_ptr.h:147
    #8 0x5649a091a6b8 in std::shared_ptr<char> dsn::utils::make_shared_array<char>(unsigned long) /home/mi/workspace/pegasus/rdsn/include/dsn/utility/utils.h:72
    #9 0x5649a0ab050d in dsn::message_reader::read_buffer_ptr(unsigned int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/message_parser.cpp:150
    #10 0x5649a0cfd678 in dsn::tools::asio_rpc_session::do_read(int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_rpc_session.cpp:83
    #11 0x5649a0ab5dff in dsn::rpc_session::start_read_next(int) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:224
    #12 0x5649a0d1309c in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_rpc_session.cpp:197
    #13 0x5649a0d1309c in operator() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/bind_handler.hpp:65
    #14 0x5649a0d1309c in asio_handler_invoke<boost::asio::detail::binder1<dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/handler_invoke_hook.hpp:69
    #15 0x5649a0d1309c in invoke<boost::asio::detail::binder1<dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)>, boost::system::error_code>, dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_invoke_helpers.hpp:37
    #16 0x5649a0d1309c in complete<boost::asio::detail::binder1<dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_work.hpp:82
    #17 0x5649a0d1309c in do_complete /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/reactive_socket_connect_op.hpp:100
    #18 0x5649a0cf9aea in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #19 0x5649a0cf9aea in boost::asio::detail::epoll_reactor::descriptor_state::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/epoll_reactor.ipp:776
    #20 0x5649a0a59c12 in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #21 0x5649a0a59c12 in boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock&, boost::asio::detail::scheduler_thread_info&, boost::system::error_code const&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:401
    #22 0x5649a0a59c12 in boost::asio::detail::scheduler::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:154
    #23 0x5649a0cdaf3f in boost::asio::io_context::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/impl/io_context.ipp:70
    #24 0x5649a0cdaf3f in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:79
    #25 0x5649a0cdaf3f in __invoke_impl<void, dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:60
    #26 0x5649a0cdaf3f in __invoke<dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:95
    #27 0x5649a0cdaf3f in _M_invoke<0> /usr/include/c++/7/thread:234
    #28 0x5649a0cdaf3f in operator() /usr/include/c++/7/thread:243
    #29 0x5649a0cdaf3f in _M_run /usr/include/c++/7/thread:186
    #30 0x7efe0ec8f6de  (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0xbd6de)

Indirect leak of 8 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a0ac657a in __gnu_cxx::new_allocator<dsn::message_ex*>::allocate(unsigned long, void const*) /usr/include/c++/7/ext/new_allocator.h:111
    #2 0x5649a0ac657a in std::allocator_traits<std::allocator<dsn::message_ex*> >::allocate(std::allocator<dsn::message_ex*>&, unsigned long) /usr/include/c++/7/bits/alloc_traits.h:436
    #3 0x5649a0ac657a in std::_Vector_base<dsn::message_ex*, std::allocator<dsn::message_ex*> >::_M_allocate(unsigned long) /usr/include/c++/7/bits/stl_vector.h:172
    #4 0x5649a0ac657a in void std::vector<dsn::message_ex*, std::allocator<dsn::message_ex*> >::_M_realloc_insert<dsn::message_ex* const&>(__gnu_cxx::__normal_iterator<dsn::message_ex**, std::vector<dsn::message_ex*, std::allocator<dsn::message_ex*> > >, dsn::message_ex* const&) /usr/include/c++/7/bits/vector.tcc:406
    #5 0x5649a0ac7061 in std::vector<dsn::message_ex*, std::allocator<dsn::message_ex*> >::push_back(dsn::message_ex* const&) /usr/include/c++/7/bits/stl_vector.h:948
    #6 0x5649a0ac7061 in dsn::rpc_session::unlink_message_for_send() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:193
    #7 0x5649a0abc1df in dsn::rpc_session::send_message(dsn::message_ex*) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:275
    #8 0x5649a0ad207e in dsn::rpc_engine::reply(dsn::message_ex*, dsn::error_code) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/rpc_engine.cpp:761
    #9 0x5649a09129b1 in dsn::rpc_replier<dsn::service::get_file_size_response>::operator()(dsn::service::get_file_size_response const&) /home/mi/workspace/pegasus/rdsn/include/dsn/cpp/serverlet.h:77
    #10 0x5649a09129b1 in dsn::service::nfs_service_impl::on_get_file_size(dsn::service::get_file_size_request const&, dsn::rpc_replier<dsn::service::get_file_size_response>&) /home/mi/workspace/pegasus/rdsn/src/nfs/nfs_server_impl.cpp:225
    #11 0x5649a090017b in bool dsn::serverlet<dsn::service::nfs_service>::register_async_rpc_handler<dsn::service::get_file_size_request, dsn::service::get_file_size_response>(dsn::task_code, char const*, void (dsn::service::nfs_service::*)(dsn::service::get_file_size_request const&, dsn::rpc_replier<dsn::service::get_file_size_response>&))::{lambda(dsn::message_ex*)#1}::operator()(dsn::message_ex*) const /home/mi/workspace/pegasus/rdsn/include/dsn/cpp/serverlet.h:217
    #12 0x5649a0b025d3 in std::function<void (dsn::message_ex*)>::operator()(dsn::message_ex*) const /usr/include/c++/7/bits/std_function.h:706
    #13 0x5649a0b025d3 in dsn::rpc_request_task::exec() /home/mi/workspace/pegasus/rdsn/include/dsn/tool-api/task.h:435
    #14 0x5649a0b08f2c in dsn::task::exec_internal() /home/mi/workspace/pegasus/rdsn/src/runtime/task/task.cpp:176
    #15 0x5649a0b6d445 in dsn::task_worker::loop() /home/mi/workspace/pegasus/rdsn/src/runtime/task/task_worker.cpp:211
    #16 0x5649a0b6e31c in dsn::task_worker::run_internal() /home/mi/workspace/pegasus/rdsn/src/runtime/task/task_worker.cpp:191
    #17 0x5649a0b6e8b3 in void std::__invoke_impl<void, void (dsn::task_worker::*&)(), dsn::task_worker*&>(std::__invoke_memfun_deref, void (dsn::task_worker::*&)(), dsn::task_worker*&) /usr/include/c++/7/bits/invoke.h:73
    #18 0x5649a0b6e8b3 in std::__invoke_result<void (dsn::task_worker::*&)(), dsn::task_worker*&>::type std::__invoke<void (dsn::task_worker::*&)(), dsn::task_worker*&>(void (dsn::task_worker::*&)(), dsn::task_worker*&) /usr/include/c++/7/bits/invoke.h:95
    #19 0x5649a0b6e8b3 in void std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>::__call<void, , 0ul>(std::tuple<>&&, std::_Index_tuple<0ul>) /usr/include/c++/7/functional:467
    #20 0x5649a0b6e8b3 in void std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>::operator()<, void>() /usr/include/c++/7/functional:551
    #21 0x5649a0b6e8b3 in void std::__invoke_impl<void, std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>>(std::__invoke_other, std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>&&) /usr/include/c++/7/bits/invoke.h:60
    #22 0x5649a0b6e8b3 in std::__invoke_result<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>>::type std::__invoke<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>>(std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>&&) /usr/include/c++/7/bits/invoke.h:95
    #23 0x5649a0b6e8b3 in decltype (__invoke((_S_declval<0ul>)())) std::thread::_Invoker<std::tuple<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()> > >::_M_invoke<0ul>(std::_Index_tuple<0ul>) /usr/include/c++/7/thread:234
    #24 0x5649a0b6e8b3 in std::thread::_Invoker<std::tuple<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()> > >::operator()() /usr/include/c++/7/thread:243
    #25 0x5649a0b6e8b3 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()> > > >::_M_run() /usr/include/c++/7/thread:186
    #26 0x7efe0ec8f6de  (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0xbd6de)

Indirect leak of 8 byte(s) in 1 object(s) allocated from:
    #0 0x7efe0fc32448 in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.4+0xe0448)
    #1 0x5649a0ac657a in __gnu_cxx::new_allocator<dsn::message_ex*>::allocate(unsigned long, void const*) /usr/include/c++/7/ext/new_allocator.h:111
    #2 0x5649a0ac657a in std::allocator_traits<std::allocator<dsn::message_ex*> >::allocate(std::allocator<dsn::message_ex*>&, unsigned long) /usr/include/c++/7/bits/alloc_traits.h:436
    #3 0x5649a0ac657a in std::_Vector_base<dsn::message_ex*, std::allocator<dsn::message_ex*> >::_M_allocate(unsigned long) /usr/include/c++/7/bits/stl_vector.h:172
    #4 0x5649a0ac657a in void std::vector<dsn::message_ex*, std::allocator<dsn::message_ex*> >::_M_realloc_insert<dsn::message_ex* const&>(__gnu_cxx::__normal_iterator<dsn::message_ex**, std::vector<dsn::message_ex*, std::allocator<dsn::message_ex*> > >, dsn::message_ex* const&) /usr/include/c++/7/bits/vector.tcc:406
    #5 0x5649a0ac7061 in std::vector<dsn::message_ex*, std::allocator<dsn::message_ex*> >::push_back(dsn::message_ex* const&) /usr/include/c++/7/bits/stl_vector.h:948
    #6 0x5649a0ac7061 in dsn::rpc_session::unlink_message_for_send() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:193
    #7 0x5649a0abd4b7 in dsn::rpc_session::on_send_completed(unsigned long) /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/network.cpp:330
    #8 0x5649a0d13076 in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_rpc_session.cpp:196
    #9 0x5649a0d13076 in operator() /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/bind_handler.hpp:65
    #10 0x5649a0d13076 in asio_handler_invoke<boost::asio::detail::binder1<dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/handler_invoke_hook.hpp:69
    #11 0x5649a0d13076 in invoke<boost::asio::detail::binder1<dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)>, boost::system::error_code>, dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_invoke_helpers.hpp:37
    #12 0x5649a0d13076 in complete<boost::asio::detail::binder1<dsn::tools::asio_rpc_session::connect()::<lambda(boost::system::error_code)>, boost::system::error_code> > /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/handler_work.hpp:82
    #13 0x5649a0d13076 in do_complete /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/reactive_socket_connect_op.hpp:100
    #14 0x5649a0cf9aea in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #15 0x5649a0cf9aea in boost::asio::detail::epoll_reactor::descriptor_state::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/epoll_reactor.ipp:776
    #16 0x5649a0a59c12 in boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/scheduler_operation.hpp:40
    #17 0x5649a0a59c12 in boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock&, boost::asio::detail::scheduler_thread_info&, boost::system::error_code const&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:401
    #18 0x5649a0a59c12 in boost::asio::detail::scheduler::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/detail/impl/scheduler.ipp:154
    #19 0x5649a0cdaf3f in boost::asio::io_context::run(boost::system::error_code&) /home/mi/workspace/pegasus/rdsn/thirdparty/output/include/boost/asio/impl/io_context.ipp:70
    #20 0x5649a0cdaf3f in operator() /home/mi/workspace/pegasus/rdsn/src/runtime/rpc/asio_net_provider.cpp:79
    #21 0x5649a0cdaf3f in __invoke_impl<void, dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:60
    #22 0x5649a0cdaf3f in __invoke<dsn::tools::asio_network_provider::start(dsn::rpc_channel, int, bool)::<lambda()> > /usr/include/c++/7/bits/invoke.h:95
    #23 0x5649a0cdaf3f in _M_invoke<0> /usr/include/c++/7/thread:234
    #24 0x5649a0cdaf3f in operator() /usr/include/c++/7/thread:243
    #25 0x5649a0cdaf3f in _M_run /usr/include/c++/7/thread:186
    #26 0x7efe0ec8f6de  (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0xbd6de)

SUMMARY: AddressSanitizer: 132288 byte(s) leaked in 14 allocation(s).
ERROR: run dsn_nfs_test failed, return_code = 1

That is because the dsn_exit is not called before, and the dsn core is already run when the unit test is done. so asan thought the memory hold by dsn core as memory leak

@levy5307 levy5307 closed this Feb 9, 2021
@levy5307 levy5307 reopened this Feb 9, 2021
@levy5307 levy5307 merged commit 124b026 into XiaoMi:master Feb 9, 2021
@levy5307 levy5307 deleted the asan-nfs branch February 9, 2021 09:32
Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Labels
None yet
Projects
None yet
Development

Successfully merging this pull request may close these issues.

3 participants