Nodeos 1.7.1 segfault ubuntu 18.04

[opcheese]

I have 2 machines running nodes for mainnet and 1 for Jungle2.

After upgrading one of mainnet nodes to 1.7.0 and subsequently to 1.7.1 I had some troubles with it. But for the last week, all is running well. However, in the last 24 hours nodeos on Jungle2 network stopped (crashed and demanded replay) 3 times. (Though it ran without trouble or changes for 2 weeks)

The first crash was a segfault. The crash file is available at http://share.ludentes.ru/_usr_opt_eosio_1.7.1_bin_nodeos.1000.crash

backtrace (gdb output):

Reading symbols from /usr/bin/nodeos...done.
[New LWP 32738]
[New LWP 32729]
[New LWP 32732]
[New LWP 32726]
[New LWP 32733]
[New LWP 32727]
[New LWP 32737]
[New LWP 32734]
[New LWP 32736]
[New LWP 32739]
[New LWP 32728]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `nodeos --data-dir . --config-dir . -l ./logging.json'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00000000004bd395 in void std::__push_heap<std::_Deque_iterator<std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >, std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >&, std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >*>, long, std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >, __gnu_cxx::__ops::_Iter_comp_val<std::less<std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> > > > >(std::_Deque_iterator<std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >, std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >&, std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >*>, long, long, std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >, __gnu_cxx::__ops::_Iter_comp_val<std::less<std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> > > >&) ()
[Current thread is 1 (Thread 0x7f8ef1ffb700 (LWP 32738))]
(gdb) bt
#0  0x00000000004bd395 in void std::__push_heap<std::_Deque_iterator<std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >, std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >&, std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >*>, long, std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >, __gnu_cxx::__ops::_Iter_comp_val<std::less<std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> > > > >(std::_Deque_iterator<std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >, std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >&, std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >*>, long, long, std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >, __gnu_cxx::__ops::_Iter_comp_val<std::less<std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> > > >&) ()
#1  0x00000000004c00df in std::priority_queue<std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >, std::deque<std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >, std::allocator<std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> > > >, std::less<std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> > > >::push(std::unique_ptr<appbase::execution_priority_queue::queued_handler_base, std::default_delete<appbase::execution_priority_queue::queued_handler_base> >&&) ()
#2  0x0000000000775f67 in boost::asio::detail::wait_handler<boost::asio::executor_binder<eosio::net_plugin_impl::start_read_message(std::shared_ptr<eosio::connection> const&)::$_13(appbase::execution_priority_queue::executor)> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long) ()
#3  0x00000000004bc088 in boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock&, boost::asio::detail::scheduler_thread_info&, boost::system::error_code const&) ()
#4  0x00000000004bfb41 in boost::asio::detail::scheduler::run(boost::system::error_code&) ()
#5  0x000000000077c683 in boost::asio::detail::executor_op<boost::asio::detail::work_dispatcher<eosio::net_plugin::plugin_startup()::$_15>, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long) ()
#6  0x00000000004bc088 in boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock&, boost::asio::detail::scheduler_thread_info&, boost::system::error_code const&) ()
#7  0x00000000004bfb41 in boost::asio::detail::scheduler::run(boost::system::error_code&) ()
#8  0x0000000000802745 in boost::asio::detail::posix_thread::func<boost::asio::thread_pool::thread_function>::run() ()
#9  0x00000000006dbe5d in boost_asio_detail_posix_thread_function ()
#10 0x00007f9f7eb7b6db in start_thread (arg=0x7f8ef1ffb700) at pthread_create.c:463
#11 0x00007f9f7dea788f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

config is available here
http://share.ludentes.ru/config.ini

Subsequent 2 crashes do not appear in kern.log And it seems like nodeos just stopped. (however, it demands replay so something definitely went wrong).

I'll keep monitoring the situation and if I have any further info I'll post here.

[matthewdarwin]

Possibly related issue: #7081

[opcheese]

Missed that one. I'll look into it. Thank you!

[heifner]

Looks very much like #7081

[heifner]

Fixed by #7085