In debug mode, the fuse process crashes in GF_ ASSERT(!(*take_lock))

[JamesWSWu]

Description of problem:
When multiple processes write to a file simultaneously without locks, the fuse process crashes

The exact command to reproduce the issue:
#!/usr/bin/python

import sys
import getopt
import random
import time
import multiprocessing

def do_help():
prompt = "python write_file.py -t <thread_num> -f <file_name>"
print(prompt)

def write_files(file_name):
while True:
#with open(file_name, "w") as f:
f = open(file_name, "w")
random.seed(time.localtime())
result = random.random()
f.write(str(result) * 1024)
#time.sleep(1)

def main(argv):
thread_num = ''
file_name = ''

try:
    opts, _ = getopt.getopt(argv, "ht:f:", ["thead_num", "file_name"])
except getopt.GetoptError:
    do_help()
    sys.exit(2)
for opt, arg in opts:
    if opt == "-h":
        do_help()
        sys.exit(0)
    elif opt in ("-t", "--thread_num"):
        thread_num = arg
    elif opt in ("-f", "--file_name"):
        file_name = arg

for num in range(int(thread_num)):
    proc = multiprocessing.Process(target=write_files, args=(file_name,))
    proc.start()

if name == "main":
main(sys.argv[1:])
while True:
time.sleep(1)

The full output of the command that failed:
The fuse process crashes. The stack information is as follows.
0x00007f4438ac03d7 in raise () from /lib64/libc.so.6
0x00007f4438ac1ac8 in abort () from /lib64/libc.so.6
0x00007f4438ab91f6 in __assert_fail_base () from /lib64/libc.so.6
0x00007f4438ab92a2 in __assert_fail () from /lib64/libc.so.6
0x00007f442b84cd9e in __afr_eager_lock_handle (local=0x7f4424091d88, take_lock=0x7f442cd3b3ae, do_pre_op=0x7f442cd3b3af, timer_local=)
at afr-transaction.c:2778
0x00007f442b84cef3 in afr_transaction_start (local=0x7f4424091d88, this=0x7f4424011880) at afr-transaction.c:2818
0x00007f442b84d3dd in afr_transaction (frame=frame@entry=0x7f442409d1a8, this=this@entry=0x7f4424011880, type=type@entry=AFR_DATA_TRANSACTION)
at afr-transaction.c:2917
0x00007f442b83680e in afr_do_writev (frame=frame@entry=0x7f4424080c58, this=this@entry=0x7f4424011880) at afr-inode-write.c:470
0x00007f442b8375f6 in afr_writev (frame=frame@entry=0x7f4424080c58, this=this@entry=0x7f4424011880, fd=fd@entry=0x7f441801b558, vector=vector@entry=0x7f442cd3b6f0,
count=count@entry=1, offset=offset@entry=0, flags=flags@entry=32769, iobref=iobref@entry=0x7f441802f320, xdata=xdata@entry=0x0) at afr-inode-write.c:549
0x00007f443a51267f in default_writev (frame=frame@entry=0x7f4424080c58, this=, fd=fd@entry=0x7f441801b558, vector=vector@entry=0x7f442cd3b6f0,
count=count@entry=1, off=off@entry=0, flags=flags@entry=32769, iobref=iobref@entry=0x7f441802f320, xdata=xdata@entry=0x0) at defaults.c:2735
0x00007f442b34ee9b in gf_utime_writev (frame=frame@entry=0x7f4424095ea8, this=0x7f44240161c0, fd=0x7f441801b558, vector=vector@entry=0x7f442cd3b6f0,
count=count@entry=1, off=0, flags=32769, iobref=0x7f441802f320, xdata=xdata@entry=0x0) at utime-autogen-fops.c:262
0x00007f442b1367d6 in wb_fulfill_head (wb_inode=wb_inode@entry=0x7f441802de90, head=0x7f441800e070) at write-behind.c:1159
0x00007f442b136a4b in wb_fulfill (wb_inode=wb_inode@entry=0x7f441802de90, liabilities=liabilities@entry=0x7f442cd3b860) at write-behind.c:1236
0x00007f442b138021 in wb_process_queue (wb_inode=wb_inode@entry=0x7f441802de90) at write-behind.c:1788
0x00007f442b1381a1 in wb_fulfill_cbk (frame=0x7f441c02ecb8, cookie=, this=, op_ret=8192, op_errno=,
prebuf=, postbuf=0x7f441c049360, xdata=0x7f442409e538) at write-behind.c:1108
0x00007f442b34acb8 in gf_utime_writev_cbk (frame=frame@entry=0x7f441c007ad8, cookie=, this=, op_ret=8192, op_errno=0,
prebuf=prebuf@entry=0x7f441c0492c8, postbuf=postbuf@entry=0x7f441c049360, xdata=0x7f442409e538) at utime-autogen-fops.c:244
0x00007f442b835f12 in afr_writev_unwind (frame=frame@entry=0x7f441c00ea78, this=this@entry=0x7f4424011880) at afr-inode-write.c:227
0x00007f442b8364de in afr_writev_wind_cbk (frame=0x7f441c004cd8, cookie=, this=0x7f4424011880, op_ret=, op_errno=,
prebuf=, postbuf=0x7f442cd3bb60, xdata=0x7f442406f548) at afr-inode-write.c:381
0x00007f442bb02008 in client4_0_writev_cbk (req=, iov=, count=, myframe=0x7f441c00aae8) at client-rpc-fops_v2.c:679
0x00007f443a229001 in rpc_clnt_handle_reply (clnt=clnt@entry=0x7f4424046880, pollin=pollin@entry=0x7f442408af10) at rpc-clnt.c:780
0x00007f443a2293bb in rpc_clnt_notify (trans=0x7f4424046c20, mydata=0x7f44240468b0, event=, data=0x7f442408af10) at rpc-clnt.c:957
0x00007f443a225a13 in rpc_transport_notify (this=this@entry=0x7f4424046c20, event=event@entry=RPC_TRANSPORT_MSG_RECEIVED, data=data@entry=0x7f442408af10)
at rpc-transport.c:530
0x00007f442e5eb408 in socket_event_poll_in_async (xl=, async=0x7f442408b028) at socket.c:2523
0x00007f442e5f0c51 in gf_async (cbk=0x7f442e5eb3e0 <socket_event_poll_in_async>, xl=, async=0x7f442408b028)
at ../../../../libglusterfs/src/glusterfs/async.h:189
socket_event_poll_in (notify_handled=true, this=0x7f4424046c20) at socket.c:2564
socket_event_handler (fd=, idx=1, gen=, data=0x7f4424046c20, poll_in=, poll_out=, poll_err=0,
event_thread_died=0 '\000') at socket.c:2955
0x00007f443a4e1eac in event_dispatch_epoll_handler (event=0x7f442cd3c130, event_pool=0x560d30d0dd60) at event-epoll.c:640
---Type to continue, or q to quit---
event_dispatch_epoll_worker (data=0x560d30d95500) at event-epoll.c:751
0x00007f44392c2e65 in start_thread () from /lib64/libpthread.so.0
0x00007f4438b8882d in clone () from /lib64/libc.so.6

Expected results:
The fuse process does not crash

Mandatory info:
- The output of the gluster volume info command:

- The output of the gluster volume status command:

- The output of the gluster volume heal command:

**- Provide logs present on following locations of client and server nodes -
[2023-03-29 08:01:02.991128] I [MSGID: 108019] [afr-transaction.c:2698:__afr_eager_lock_handle] 0-f798231e-d3a0-41af-9e82-6979e420b8d9-replicate-0: =============== task_lock: 1
[2023-03-29 08:01:02.991189] I [MSGID: 108019] [afr-transaction.c:2772:__afr_eager_lock_handle] 0-f798231e-d3a0-41af-9e82-6979e420b8d9-replicate-0: =============== task_lock: 1
The message "I [MSGID: 108019] [afr-transaction.c:2737:__afr_eager_lock_handle] 0-f798231e-d3a0-41af-9e82-6979e420b8d9-replicate-0: =============== " repeated 137 times between [2023-03-29 08:00:56.494355] and [2023-03-29 08:01:02.990064]
pending frames:
frame : type(0) op(0)
frame : type(0) op(0)
frame : type(0) op(0)
frame : type(0) op(0)
frame : type(0) op(0)
frame : type(1) op(FSTAT)
frame : type(1) op(FSTAT)
frame : type(1) op(TRUNCATE)
frame : type(1) op(FLUSH)
frame : type(1) op(FLUSH)
frame : type(1) op(FSTAT)
frame : type(1) op(FLUSH)
frame : type(1) op(FSTAT)
patchset: git://git.gluster.org/glusterfs.git
signal received: 6
time of crash:
2023-03-29 08:01:02
configuration details:
argp 1
backtrace 1
dlfcn 1
libpthread 1
llistxattr 1
setfsid 1
spinlock 1
epoll.h 1
xattr.h 1
st_atim.tv_nsec 1
package-string: glusterfs 8.2
/lib64/libglusterfs.so.0(+0x28b5f)[0x7fa63f68db5f]
/lib64/libglusterfs.so.0(gf_print_trace+0x334)[0x7fa63f697ca4]
/lib64/libc.so.6(+0x36450)[0x7fa63dccf450]
/lib64/libc.so.6(gsignal+0x37)[0x7fa63dccf3d7]
/lib64/libc.so.6(abort+0x148)[0x7fa63dcd0ac8]
/lib64/libc.so.6(+0x2f1f6)[0x7fa63dcc81f6]
/lib64/libc.so.6(+0x2f2a2)[0x7fa63dcc82a2]
/usr/lib64/glusterfs/8.2/xlator/cluster/replicate.so(+0x3ed86)[0x7fa630a5bd86]
/usr/lib64/glusterfs/8.2/xlator/cluster/replicate.so(+0x3ee93)[0x7fa630a5be93]
/usr/lib64/glusterfs/8.2/xlator/cluster/replicate.so(+0x3f37d)[0x7fa630a5c37d]
/usr/lib64/glusterfs/8.2/xlator/cluster/replicate.so(+0x2880e)[0x7fa630a4580e]
/usr/lib64/glusterfs/8.2/xlator/cluster/replicate.so(+0x295f6)[0x7fa630a465f6]
/lib64/libglusterfs.so.0(default_writev+0x10f)[0x7fa63f72167f]
/usr/lib64/glusterfs/8.2/xlator/features/utime.so(+0x8e9b)[0x7fa63055de9b]
/usr/lib64/glusterfs/8.2/xlator/performance/write-behind.so(+0x87d6)[0x7fa6303457d6]
/usr/lib64/glusterfs/8.2/xlator/performance/write-behind.so(+0x8a4b)[0x7fa630345a4b]

**- Is there any crash ? Provide the backtrace and coredump

Additional info:

- The operating system / glusterfs version:

Note: Please hide any confidential data which you don't want to share in public like IP address, file name, hostname or any other configuration