system_tests_delivery_counts fails with use after free of qd_message_t

[ganeshmurthy]

____________ ERROR at teardown of TwoRouterLargeMessagePresettledReleasedCountTest.test_verify_inter_router_presettled_released_count_DISPATCH_1541 ____________

self = <system_test.Tester object at 0x7f0daf4b7ed0>

    def teardown(self):
        """Clean up (tear-down, stop or close) objects recorded via cleanup()"""
        self.cleanup_list.reverse()
        errors = []
        for obj in self.cleanup_list:
            try:
                for method in ["teardown", "tearDown", "stop", "close"]:
                    cleanup = getattr(obj, method, None)
                    if cleanup:
>                       cleanup()

/home/runner/work/skupper-router/skupper-router/skupper-router/tests/system_test.py:1204: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/home/runner/work/skupper-router/skupper-router/skupper-router/tests/system_test.py:827: in teardown
    raise teardown_exc
/home/runner/work/skupper-router/skupper-router/skupper-router/tests/system_test.py:765: in teardown
    super(Qdrouterd, self).teardown()
/home/runner/work/skupper-router/skupper-router/skupper-router/tests/system_test.py:440: in teardown
    error("returned error code %s" % self.returncode)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

msg = 'returned error code 1'

    def error(msg):
        with open(self.outfile_path) as f:
>           raise RuntimeError("Process %s (name=%s) error: %s\n%s\n%s\n>>>>\n%s<<<<" % (
                self.pid, self.name, msg, ' '.join(self.args),
                self.cmdfile_path, f.read()))
E           RuntimeError: Process 4565 (name=A) error: returned error code 1
E           skrouterd -c A.conf -I /home/runner/work/skupper-router/skupper-router/skupper-router/python
E           /__w/skupper-router/skupper-router/skupper-router/build/tests/system_test.dir/tests/system_tests_delivery_counts/TwoRouterLargeMessagePresettledReleasedCountTest/setUpClass/A-8.cmd
E           >>>>
E           =================================================================
E           ==4565==ERROR: AddressSanitizer: use-after-poison on address 0x61200020dfe0 at pc 0x0000005ff3f1 bp 0x7ff96a386c00 sp 0x7ff96a386bf8
E           READ of size 8 at 0x61200020dfe0 thread T5
E               #0 0x5ff3f0 in qd_message_send /home/runner/work/skupper-router/skupper-router/skupper-router/src/message.c:1899
E               #1 0x81b8a7 in AMQP_conn_wake_handler /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_node.c:396
E               #2 0x81efde in AMQP_link_flow_handler /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_node.c:1195
E               #3 0x84b2d1 in handle /home/runner/work/skupper-router/skupper-router/skupper-router/src/server.c:1105
E               #4 0x8510a4 in thread_run /home/runner/work/skupper-router/skupper-router/skupper-router/src/server.c:1143
E               #5 0x8510a4 in thread_run /home/runner/work/skupper-router/skupper-router/skupper-router/src/server.c:1110
E               #6 0x7ff973a7c15c in start_thread (/lib64/libc.so.6+0x8b15c)
E               #7 0x7ff973afdbff in __clone3 (/lib64/libc.so.6+0x10cbff)
E           
E           0x61200020dfe0 is located 160 bytes inside of 256-byte region [0x61200020df40,0x61200020e040)
E           allocated by thread T2 here:
E               #0 0x7ff975138cdd in __interceptor_posix_memalign (/lib64/libasan.so.8+0xb9cdd)
E               #1 0x553895 in qd_alloc /home/runner/work/skupper-router/skupper-router/skupper-router/src/alloc_pool.c:343
E               #2 0x6001c8 in new_qd_message_t /home/runner/work/skupper-router/skupper-router/skupper-router/src/message.c:95
E               #3 0x6001c8 in qd_message_copy /home/runner/work/skupper-router/skupper-router/skupper-router/src/message.c:1163
E               #4 0x72d3af in qdr_forward_new_delivery_CT /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/forwarder.c:175
E               #5 0x756138 in qdr_forward_closest_CT /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/forwarder.c:885
E               #6 0x80d273 in qdr_forward_message_CT /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/forwarder.c:1165
E               #7 0x80d273 in qcm_mobile_sync_on_mar_CT /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/modules/mobile_sync/mobile.c:540
E               #8 0x80d273 in qcm_mobile_sync_on_message_CT /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/modules/mobile_sync/mobile.c:789
E               #9 0x72e990 in qdr_forward_on_message_CT /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/forwarder.c:415
E               #10 0x74e0c6 in qdr_forward_to_subscriber_CT /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/forwarder.c:535
E               #11 0x750a1b in qdr_forward_multicast_CT /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/forwarder.c:695
E               #12 0x7ac1e7 in qdr_forward_message_CT /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/forwarder.c:1165
E               #13 0x7ac1e7 in qdr_link_forward_CT /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/transfer.c:587
E               #14 0x7b30e3 in qdr_link_deliver_CT /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/transfer.c:880
E               #15 0x7b30e3 in qdr_link_deliver_CT /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/transfer.c:732
E               #16 0x774eb8 in router_core_thread /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/router_core_thread.c:255
E               #17 0x7ff973a7c15c in start_thread (/lib64/libc.so.6+0x8b15c)
E           
E           Thread T5 created by T0 here:
E               #0 0x7ff9750ca3e6 in __interceptor_pthread_create (/lib64/libasan.so.8+0x4b3e6)
E               #1 0x639aa4 in sys_thread /home/runner/work/skupper-router/skupper-router/skupper-router/src/posix/threading.c:228
E               #2 0x85330d in qd_server_run /home/runner/work/skupper-router/skupper-router/skupper-router/src/server.c:1514
E               #3 0x44b6f6 in main_process /home/runner/work/skupper-router/skupper-router/skupper-router/router/src/main.c:111
E               #4 0x43c698 in main /home/runner/work/skupper-router/skupper-router/skupper-router/router/src/main.c:365
E               #5 0x7ff973a1850f in __libc_start_call_main (/lib64/libc.so.6+0x2750f)
E           
E           Thread T2 created by T0 here:
E               #0 0x7ff9750ca3e6 in __interceptor_pthread_create (/lib64/libasan.so.8+0x4b3e6)
E               #1 0x639aa4 in sys_thread /home/runner/work/skupper-router/skupper-router/skupper-router/src/posix/threading.c:228
E               #2 0x77e1c0 in qdr_core /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_core/router_core.c:125
E               #3 0x82b2c2 in qd_router_setup_late /home/runner/work/skupper-router/skupper-router/skupper-router/src/router_node.c:2321
E               #4 0x7ff970f39a05 in ffi_call_unix64 (/lib64/libffi.so.8+0x7a05)
E               #5 0x7ffe4fbf5dff  ([stack]+0x1edff)
E           
E           SUMMARY: AddressSanitizer: use-after-poison /home/runner/work/skupper-router/skupper-router/skupper-router/src/message.c:1899 in qd_message_send
E           Shadow bytes around the buggy address:
E             0x0c2480039ba0: 00 00 00 00 00 00 00 00 fa fa fa fa fa fa fa fa
E             0x0c2480039bb0: fa fa fa fa fa fa fa fa 00 00 00 00 00 00 00 00
E             0x0c2480039bc0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
E             0x0c2480039bd0: 00 00 00 00 00 00 00 00 fa fa fa fa fa fa fa fa
E             0x0c2480039be0: fa fa fa fa fa fa fa fa 00 00 00 00 00 00 00 00
E           =>0x0c2480039bf0: 00 00 00 00 00 00 00 00 00 00 00 00[00]00 00 00
E             0x0c2480039c00: 00 00 00 00 00 00 00 00 fa fa fa fa fa fa fa fa
E             0x0c2480039c10: fa fa fa fa fa fa fa fa 00 00 00 00 00 00 00 00
E             0x0c2480039c20: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
E             0x0c2480039c30: 00 00 00 00 00 00 00 00 fa fa fa fa fa fa fa fa
E             0x0c2480039c40: fa fa fa fa fa fa fa fa 00 00 00 00 00 00 00 00
E           Shadow byte legend (one shadow byte represents 8 application bytes):
E             Addressable:           00
E             Partially addressable: 01 02 03 04 05 06 07 
E             Heap left redzone:       fa
E             Freed heap region:       fd
E             Stack left redzone:      f1
E             Stack mid redzone:       f2
E             Stack right redzone:     f3
E             Stack after return:      f5
E             Stack use after scope:   f8
E             Global redzone:          f9
E             Global init order:       f6
E             Poisoned by user:        f7
E             Container overflow:      fc
E             Array cookie:            ac
E             Intra object redzone:    bb
E             ASan internal:           fe
E             Left alloca redzone:     ca
E             Right alloca redzone:    cb
E           ==4565==ABORTING
E           <<<<

/home/runner/work/skupper-router/skupper-router/skupper-router/tests/system_test.py:405: RuntimeError

The above exception was the direct cause of the following exception:

https://github.com/skupperproject/skupper-router/actions/runs/6239621427/job/16937923417?pr=1219#step:34:4760