feat: ipc: add nested-free flag support to mutex

This patch introduces a `nested-free` control flag to the mutex,
allowing more flexible control over mutex hold limits. The change is necessary
to handle scenarios where nested mutex acquisition is restricted, since
it will introduce possibility of data corruption in a nested access to
the critical area.

Changes:
- Added `ctrl_flags` field in `rt_mutex` structure to replace the reserved
  field for holding control flags.
- Introduced `RT_MUTEX_CTRL_NESTED_FREE` command in `rt_mutex_control` to
  enable the new flag.
- Refactored mutex initialization by extracting common logic into a new
  `_mutex_init` function, reducing code duplication.
- Updated the `_rt_mutex_take` function to respect the `nested-free` flag,
  preventing nested acquisition when the flag is set.
- Modified `rt_mutex_control` to handle the new control command and return
  appropriate error codes for invalid commands.

Signed-off-by: Shell <[email protected]>

include/rtdef.h

@@ -1028,7 +1028,7 @@ struct rt_mutex
     rt_uint8_t           ceiling_priority;              /**< the priority ceiling of mutexe */
     rt_uint8_t           priority;                      /**< the maximal priority for pending thread */
     rt_uint8_t           hold;                          /**< numbers of thread hold the mutex */
-    rt_uint8_t           reserved;                      /**< reserved field */
+    rt_uint8_t           ctrl_flags;                    /**< flags to control mutex behavior */
 
     struct rt_thread    *owner;                         /**< current owner of mutex */
     rt_list_t            taken_list;                    /**< the object list taken by thread */

include/rtthread.h

@@ -445,13 +445,18 @@ rt_err_t rt_sem_control(rt_sem_t sem, int cmd, void *arg);
  */
 
 #ifdef RT_USING_MUTEX
+
+#define RT_MTX_CTRL_FLAG_DEFAULT    0x0     /* create a mutex with default behavior */
+#define RT_MTX_CTRL_FLAG_RESERVED   0x01    /* RESERVED for legacy IPC flag */
+RT_STATIC_ASSERT(ident_to_ipc_flag, RT_MTX_CTRL_FLAG_RESERVED == RT_IPC_FLAG_PRIO);
+#define RT_MTX_CTRL_FLAG_NO_RECUR   0x02    /* create a mutex without recursion taken */
 /*
  * mutex interface
  */
-rt_err_t rt_mutex_init(rt_mutex_t mutex, const char *name, rt_uint8_t flag);
+rt_err_t rt_mutex_init(rt_mutex_t mutex, const char *name, rt_uint8_t ctrl_flag);
 rt_err_t rt_mutex_detach(rt_mutex_t mutex);
 #ifdef RT_USING_HEAP
-rt_mutex_t rt_mutex_create(const char *name, rt_uint8_t flag);
+rt_mutex_t rt_mutex_create(const char *name, rt_uint8_t ctrl_flag);
 rt_err_t rt_mutex_delete(rt_mutex_t mutex);
 #endif /* RT_USING_HEAP */
 void rt_mutex_drop_thread(rt_mutex_t mutex, rt_thread_t thread);

src/ipc.c

@@ -980,6 +980,23 @@ static void _mutex_before_delete_detach(rt_mutex_t mutex)
  * @{
  */
 
+static void _mutex_init(rt_mutex_t mutex, rt_uint8_t ctrl_flags)
+{
+    /* initialize ipc object */
+    _ipc_object_init(&(mutex->parent));
+
+    mutex->owner    = RT_NULL;
+    mutex->priority = 0xFF;
+    mutex->hold     = 0;
+    mutex->ceiling_priority = 0xFF;
+    mutex->ctrl_flags = ctrl_flags;
+    rt_list_init(&(mutex->taken_list));
+
+    /* flag can only be RT_IPC_FLAG_PRIO. RT_IPC_FLAG_FIFO cannot solve the unbounded priority inversion problem */
+    mutex->parent.parent.flag = RT_IPC_FLAG_PRIO;
+    rt_spin_lock_init(&(mutex->spinlock));
+}
+
 /**
  * @brief    Initialize a static mutex object.
  *
@@ -1004,29 +1021,15 @@ static void _mutex_before_delete_detach(rt_mutex_t mutex)
  *
  * @warning  This function can ONLY be called from threads.
  */
-rt_err_t rt_mutex_init(rt_mutex_t mutex, const char *name, rt_uint8_t flag)
+rt_err_t rt_mutex_init(rt_mutex_t mutex, const char *name, rt_uint8_t ctrl_flag)
 {
-    /* flag parameter has been obsoleted */
-    RT_UNUSED(flag);
-
     /* parameter check */
     RT_ASSERT(mutex != RT_NULL);
 
     /* initialize object */
     rt_object_init(&(mutex->parent.parent), RT_Object_Class_Mutex, name);
 
-    /* initialize ipc object */
-    _ipc_object_init(&(mutex->parent));
-
-    mutex->owner    = RT_NULL;
-    mutex->priority = 0xFF;
-    mutex->hold     = 0;
-    mutex->ceiling_priority = 0xFF;
-    rt_list_init(&(mutex->taken_list));
-
-    /* flag can only be RT_IPC_FLAG_PRIO. RT_IPC_FLAG_FIFO cannot solve the unbounded priority inversion problem */
-    mutex->parent.parent.flag = RT_IPC_FLAG_PRIO;
-    rt_spin_lock_init(&(mutex->spinlock));
+    _mutex_init(mutex, ctrl_flag);
 
     return RT_EOK;
 }
@@ -1230,32 +1233,18 @@ RTM_EXPORT(rt_mutex_getprioceiling);
  *
  * @warning  This function can ONLY be called from threads.
  */
-rt_mutex_t rt_mutex_create(const char *name, rt_uint8_t flag)
+rt_mutex_t rt_mutex_create(const char *name, rt_uint8_t ctrl_flags)
 {
     struct rt_mutex *mutex;
 
-    /* flag parameter has been obsoleted */
-    RT_UNUSED(flag);
-
     RT_DEBUG_NOT_IN_INTERRUPT;
 
     /* allocate object */
     mutex = (rt_mutex_t)rt_object_allocate(RT_Object_Class_Mutex, name);
     if (mutex == RT_NULL)
         return mutex;
 
-    /* initialize ipc object */
-    _ipc_object_init(&(mutex->parent));
-
-    mutex->owner    = RT_NULL;
-    mutex->priority = 0xFF;
-    mutex->hold     = 0;
-    mutex->ceiling_priority = 0xFF;
-    rt_list_init(&(mutex->taken_list));
-
-    /* flag can only be RT_IPC_FLAG_PRIO. RT_IPC_FLAG_FIFO cannot solve the unbounded priority inversion problem */
-    mutex->parent.parent.flag = RT_IPC_FLAG_PRIO;
-    rt_spin_lock_init(&(mutex->spinlock));
+    _mutex_init(mutex, ctrl_flags);
 
     return mutex;
 }
@@ -1351,7 +1340,8 @@ static rt_err_t _rt_mutex_take(rt_mutex_t mutex, rt_int32_t timeout, int suspend
 
     if (mutex->owner == thread)
     {
-        if (mutex->hold < RT_MUTEX_HOLD_MAX)
+        if (!(mutex->ctrl_flags & RT_MUTEX_CTRL_NESTED_FREE) &&
+            mutex->hold < RT_MUTEX_HOLD_MAX)
         {
             /* it's the same thread */
             mutex->hold ++;