replica server with security enabled exited abnormally due to blob::create_from_bytes assertion

[empiredan]

Replica server was started with security enabled as following configurations:

[security]
enable_auth = true
krb5_keytab = ...
krb5_config = /etc/krb5.conf
krb5_principal = ...
sasl_plugin_path = /usr/lib64/sasl2
service_fqdn = ...
service_name = ...
mandatory_auth = true
enable_acl = true
super_users =
meta_acl_rpc_allow_list =
enable_ranger_acl = true

However, it failed immediately after started with following error logs and generate core dump file:

E2024-09-04 12:06:36.870 (1725422796870535314 3e18) replica.default2.0201000500000008: compaction_filter_rule.cpp:109:create_from_bytes(): assertion expression: (s) != nullptr
F2024-09-04 12:06:36.870 (1725422796870576704 3e18) replica.default2.0201000500000008: compaction_filter_rule.cpp:109:create_from_bytes(): null source pointer would lead to undefined behaviour

The backtrace of core dump file is:

#0  0x00007f6375ec7387 in raise () from /lib64/libc.so.6
#1  0x00007f6375ec8a78 in abort () from /lib64/libc.so.6
#2  0x00007f637ae8b716 in dsn_coredump () at /data/code/pegasus-2.4/src/rdsn/src/runtime/service_api_c.cpp:95
#3  0x00007f6379af0e39 in dsn::tools::(anonymous namespace)::process_fatal_log (log_level=LOG_LEVEL_FATAL) at /data/code/pegasus-2.4/src/rdsn/src/utils/simple_logger.cpp:91
#4  0x00007f6379af2293 in dsn::tools::simple_logger::dsn_log (this=0x2ad8e70, file=0xcb99e5 "compaction_filter_rule.cpp",
    function=0xcbc730 <dsn::blob::create_from_bytes(char const*, unsigned long)::__FUNCTION__> "create_from_bytes", line=109, log_level=LOG_LEVEL_FATAL,
    str=0x6e667e8 "null source pointer would lead to undefined behaviour") at /data/code/pegasus-2.4/src/rdsn/src/utils/simple_logger.cpp:334
#5  0x00007f6379abf2df in dsn_log (file=0xcb99e5 "compaction_filter_rule.cpp",
    function=0xcbc730 <dsn::blob::create_from_bytes(char const*, unsigned long)::__FUNCTION__> "create_from_bytes", line=109, log_level=LOG_LEVEL_FATAL,
    str=0x6e667e8 "null source pointer would lead to undefined behaviour") at /data/code/pegasus-2.4/src/rdsn/src/utils/logging.cpp:155
#6  0x0000000000728b7e in dsn::blob::create_from_bytes (s=0x0, len=0) at /data/code/pegasus-2.4/src/rdsn/include/dsn/utility/blob.h:109
#7  0x00007f637b02d876 in dsn::security::sasl_client_wrapper::step (this=0x6aaad80, input=..., output=...) at /data/code/pegasus-2.4/src/rdsn/src/runtime/security/sasl_client_wrapper.cpp:69
#8  0x00007f637afe2ee6 in dsn::security::client_negotiation::on_challenge (this=0x677e510, challenge=...) at /data/code/pegasus-2.4/src/rdsn/src/runtime/security/client_negotiation.cpp:154
#9  0x00007f637afe26b6 in dsn::security::client_negotiation::handle_response(dsn::error_code, dsn::security::negotiation_response const&&) (this=0x677e510, err=...,
    response=<unknown type in /data/pegasus/replica_server/lib/libdsn_replica_server.so, CU 0x59bb2b0, DIE 0x5a5bc61>)
    at /data/code/pegasus-2.4/src/rdsn/src/runtime/security/client_negotiation.cpp:80
#10 0x00007f637aee0916 in dsn::security::negotiation_manager::on_negotiation_response (err=..., rpc=...) at /data/code/pegasus-2.4/src/rdsn/src/runtime/security/negotiation_manager.cpp:170
#11 0x00007f637afe3212 in dsn::security::client_negotiation::<lambda(dsn::error_code)>::operator()(dsn::error_code) (__closure=0x6e96260, err=...)
    at /data/code/pegasus-2.4/src/rdsn/src/runtime/security/client_negotiation.cpp:191
#12 0x00007f637afe3520 in dsn::rpc_holder<dsn::security::negotiation_request, dsn::security::negotiation_response>::<lambda(dsn::error_code, dsn::message_ex*, dsn::message_ex*)>::operator()(dsn::error_code, dsn::message_ex *, dsn::message_ex *) (__closure=0x6e96260, err=..., req=0x67806e0, resp=0x6780370) at /data/code/pegasus-2.4/src/rdsn/include/dsn/cpp/rpc_holder.h:161
#13 0x00007f637afe38f7 in std::_Function_handler<void(dsn::error_code, dsn::message_ex*, dsn::message_ex*), dsn::rpc_holder<TRequest, TResponse>::call(const dsn::rpc_address&, dsn::task_tracker*, TCallback&&, int) [with TCallback = dsn::security::client_negotiation::send(dsn::security::negotiation_status::type, const dsn::blob&)::<lambda(dsn::error_code)>; TRequest = dsn::security::negotiation_request; TResponse = dsn::security::negotiation_response; dsn::task_ptr = dsn::ref_ptr<dsn::task>]::<lambda(dsn::error_code, dsn::message_ex*, dsn::message_ex*)> >::_M_invoke(const std::_Any_data &, <unknown type in /data/pegasus/replica_server/lib/libdsn_replica_server.so, CU 0x59bb2b0, DIE 0x5a5022b>, <unknown type in /data/pegasus/replica_server/lib/libdsn_replica_server.so, CU 0x59bb2b0, DIE 0x5a5023a>, <unknown type in /data/pegasus/replica_server/lib/libdsn_replica_server.so, CU 0x59bb2b0, DIE 0x5a5024a>) (__functor=...,
    __args#0=<unknown type in /data/pegasus/replica_server/lib/libdsn_replica_server.so, CU 0x59bb2b0, DIE 0x5a5022b>,
    __args#1=<unknown type in /data/pegasus/replica_server/lib/libdsn_replica_server.so, CU 0x59bb2b0, DIE 0x5a5023a>,
    __args#2=<unknown type in /data/pegasus/replica_server/lib/libdsn_replica_server.so, CU 0x59bb2b0, DIE 0x5a5024a>)
    at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:316
#14 0x0000000000be5003 in std::function<void (dsn::error_code, dsn::message_ex*, dsn::message_ex*)>::operator()(dsn::error_code, dsn::message_ex*, dsn::message_ex*) const (this=0x4c0b1e0,
    __args#0=..., __args#1=0x67806e0, __args#2=0x6780370) at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:706
#15 0x00007f637af2857b in dsn::rpc_response_task::exec (this=0x4c0b100) at /data/code/pegasus-2.4/src/rdsn/include/dsn/tool-api/task.h:478
#16 0x00007f637af29bf4 in dsn::task::exec_internal (this=0x4c0b100) at /data/code/pegasus-2.4/src/rdsn/src/runtime/task/task.cpp:176
#17 0x00007f637af5a7f3 in dsn::task_worker::loop (this=0x2ad96b0) at /data/code/pegasus-2.4/src/rdsn/src/runtime/task/task_worker.cpp:224
#18 0x00007f637af5a741 in dsn::task_worker::run_internal (this=0x2ad96b0) at /data/code/pegasus-2.4/src/rdsn/src/runtime/task/task_worker.cpp:204
#19 0x00007f637af5d78a in std::__invoke_impl<void, void (dsn::task_worker::*&)(), dsn::task_worker*&> (
    __f=@0x33965e8: (void (dsn::task_worker::*)(dsn::task_worker * const)) 0x7f637af5a53a <dsn::task_worker::run_internal()>, __t=@0x33965f8: 0x2ad96b0)
    at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/invoke.h:73
#20 0x00007f637af5d679 in std::__invoke<void (dsn::task_worker::*&)(), dsn::task_worker*&> (
    __fn=@0x33965e8: (void (dsn::task_worker::*)(dsn::task_worker * const)) 0x7f637af5a53a <dsn::task_worker::run_internal()>, __args#0=@0x33965f8: 0x2ad96b0)
    at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/invoke.h:95
#21 0x00007f637af5d4c6 in std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>::__call<void, , 0ul>(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x33965e8,
    __args=<unknown type in /data/pegasus/replica_server/lib/libdsn_replica_server.so, CU 0x51576e8, DIE 0x51b9cdc>)
    at /opt/rh/devtoolset-7/root/usr/include/c++/7/functional:467
#22 0x00007f637af5d252 in std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>::operator()<, void>() (this=0x33965e8) at /opt/rh/devtoolset-7/root/usr/include/c++/7/functional:551
#23 0x00007f637af5cf61 in std::__invoke_impl<void, std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>>(std::__invoke_other, std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>&&) (__f=<unknown type in /data/pegasus/replica_server/lib/libdsn_replica_server.so, CU 0x51576e8, DIE 0x51bde89>)
    at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/invoke.h:60
#24 0x00007f637af5cb5c in std::__invoke<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>>(std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()>&&) (
    __fn=<unknown type in /data/pegasus/replica_server/lib/libdsn_replica_server.so, CU 0x51576e8, DIE 0x51c096d>)
    at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/invoke.h:95
#25 0x00007f637af5d97e in std::thread::_Invoker<std::tuple<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()> > >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x33965e8)
    at /opt/rh/devtoolset-7/root/usr/include/c++/7/thread:234
---Type <return> to continue, or q <return> to quit---f
#26 0x00007f637af5d94f in std::thread::_Invoker<std::tuple<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()> > >::operator()() (this=0x33965e8)
    at /opt/rh/devtoolset-7/root/usr/include/c++/7/thread:243
#27 0x00007f637af5d92e in std::thread::_State_impl<std::thread::_Invoker<std::tuple<std::_Bind<void (dsn::task_worker::*(dsn::task_worker*))()> > > >::_M_run() (this=0x33965e0)
    at /opt/rh/devtoolset-7/root/usr/include/c++/7/thread:186
#28 0x00007f6379b4499f in execute_native_thread_routine () from /data/pegasus/replica_server/lib/libdsn_utils.so
#29 0x00007f6377ab7ea5 in start_thread () from /lib64/libpthread.so.0
#30 0x00007f6375f8fb0d in clone () from /lib64/libc.so.6

The reason is that blob::create_from_bytes is not allowed to accept nullptr:

 58 error_s sasl_client_wrapper::step(const blob &input, blob &output)
 59 {
 60     FAIL_POINT_INJECT_F("sasl_client_wrapper_step", [](dsn::string_view str) {
 61         error_code err = error_code::try_get(str.data(), ERR_UNKNOWN);
 62         return error_s::make(err);
 63     });
 64
 65     const char *msg = nullptr;
 66     unsigned msg_len = 0;
 67     int sasl_err = sasl_client_step(_conn, input.data(), input.length(), nullptr, &msg, &msg_len);
 68
 69     output = blob::create_from_bytes(msg, msg_len);
 70     return wrap_error(sasl_err);
 71 }

105     /// Create shared buffer from allocated raw bytes.
106     /// NOTE: this operation is not efficient since it involves a memory copy.
107     [[nodiscard]] static blob create_from_bytes(const char *s, size_t len)
108     {
109         dcheck_notnull(s, "null source pointer would lead to undefined behaviour");
110
111         std::shared_ptr<char> s_arr(new char[len], std::default_delete<char[]>());
112         memcpy(s_arr.get(), s, len);
113         return {std::move(s_arr), static_cast<unsigned int>(len)};
114     }