diff --git a/include/urcu/rculfhash.h b/include/urcu/rculfhash.h index 29dd88f1..c18cf529 100644 --- a/include/urcu/rculfhash.h +++ b/include/urcu/rculfhash.h @@ -237,16 +237,16 @@ struct cds_lfht *cds_lfht_new(unsigned long init_size, * * Return 0 on success, negative error value on error. + * Threads calling this API need to be registered RCU read-side threads. + * * Prior to liburcu 0.10: - * - Threads calling this API need to be registered RCU read-side - * threads. * - cds_lfht_destroy should *not* be called from a RCU read-side * critical section. It should *not* be called from a call_rcu thread * context neither. * * Starting from liburcu 0.10, rculfhash implements its own worker - * thread to handle resize operations, which removes RCU requirements on - * cds_lfht_destroy. + * thread to handle resize operations, which removes the above RCU + * read-side critical section requirement on cds_lfht_destroy. */ extern int cds_lfht_destroy(struct cds_lfht *ht, pthread_attr_t **attr); diff --git a/src/rculfhash-internal.h b/src/rculfhash-internal.h index d29a9232..68c43746 100644 --- a/src/rculfhash-internal.h +++ b/src/rculfhash-internal.h @@ -29,6 +29,8 @@ #include #include +#include "workqueue.h" + #ifdef DEBUG #define dbg_printf(fmt, args...) printf("[debug rculfhash] " fmt, ## args) #else @@ -82,11 +84,13 @@ struct cds_lfht { * therefore cause grace-period deadlock if we hold off RCU G.P. * completion. */ - pthread_mutex_t resize_mutex; /* resize mutex: add/del mutex */ - pthread_attr_t *resize_attr; /* Resize threads attributes */ + pthread_mutex_t resize_mutex; /* resize mutex: add/del mutex */ + pthread_attr_t *caller_resize_attr; /* resize threads attributes from lfht_new caller */ + pthread_attr_t resize_attr; unsigned int in_progress_destroy; unsigned long resize_target; int resize_initiated; + struct urcu_work destroy_work; /* * Variables needed for add and remove fast-paths. diff --git a/src/rculfhash.c b/src/rculfhash.c index 9209d5f9..ef80ffa9 100644 --- a/src/rculfhash.c +++ b/src/rculfhash.c @@ -363,7 +363,6 @@ struct partition_resize_work { }; static struct urcu_workqueue *cds_lfht_workqueue; -static unsigned long cds_lfht_workqueue_user_count; /* * Mutex ensuring mutual exclusion between workqueue initialization and @@ -380,8 +379,8 @@ static struct urcu_atfork cds_lfht_atfork; */ static int cds_lfht_workqueue_atfork_nesting; +static void __attribute__((destructor)) cds_lfht_exit(void); static void cds_lfht_init_worker(const struct rcu_flavor_struct *flavor); -static void cds_lfht_fini_worker(const struct rcu_flavor_struct *flavor); #ifdef CONFIG_CDS_LFHT_ITER_DEBUG @@ -1273,7 +1272,8 @@ void partition_resize_helper(struct cds_lfht *ht, unsigned long i, work[thread].len = partition_len; work[thread].start = thread * partition_len; work[thread].fct = fct; - ret = pthread_create(&(work[thread].thread_id), ht->resize_attr, + ret = pthread_create(&(work[thread].thread_id), + ht->caller_resize_attr ? &ht->resize_attr : NULL, partition_resize_thread, &work[thread]); if (ret == EAGAIN) { /* @@ -1620,7 +1620,9 @@ struct cds_lfht *_cds_lfht_new(unsigned long init_size, ht->flags = flags; ht->flavor = flavor; - ht->resize_attr = attr; + ht->caller_resize_attr = attr; + if (attr) + ht->resize_attr = *attr; alloc_split_items_count(ht); /* this mutex should not nest in read-side C.S. */ pthread_mutex_init(&ht->resize_mutex, NULL); @@ -1834,6 +1836,35 @@ int cds_lfht_is_node_deleted(const struct cds_lfht_node *node) return is_removed(CMM_LOAD_SHARED(node->next)); } +static +bool cds_lfht_is_empty(struct cds_lfht *ht) +{ + struct cds_lfht_node *node, *next; + bool empty = true; + bool was_online; + + was_online = ht->flavor->read_ongoing(); + if (!was_online) { + ht->flavor->thread_online(); + ht->flavor->read_lock(); + } + /* Check that the table is empty */ + node = bucket_at(ht, 0); + do { + next = rcu_dereference(node->next); + if (!is_bucket(next)) { + empty = false; + break; + } + node = clear_flag(next); + } while (!is_end(node)); + if (!was_online) { + ht->flavor->read_unlock(); + ht->flavor->thread_offline(); + } + return empty; +} + static int cds_lfht_delete_bucket(struct cds_lfht *ht) { @@ -1868,6 +1899,24 @@ int cds_lfht_delete_bucket(struct cds_lfht *ht) return 0; } +static +void do_auto_resize_destroy_cb(struct urcu_work *work) +{ + struct cds_lfht *ht = caa_container_of(work, struct cds_lfht, destroy_work); + int ret; + + ht->flavor->register_thread(); + ret = cds_lfht_delete_bucket(ht); + if (ret) + urcu_die(ret); + free_split_items_count(ht); + ret = pthread_mutex_destroy(&ht->resize_mutex); + if (ret) + urcu_die(ret); + ht->flavor->unregister_thread(); + poison_free(ht); +} + /* * Should only be called when no more concurrent readers nor writers can * possibly access the table. @@ -1877,22 +1926,38 @@ int cds_lfht_destroy(struct cds_lfht *ht, pthread_attr_t **attr) int ret; if (ht->flags & CDS_LFHT_AUTO_RESIZE) { + /* + * Perform error-checking for emptiness before queuing + * work, so we can return error to the caller. This runs + * concurrently with ongoing resize. + */ + if (!cds_lfht_is_empty(ht)) + return -EPERM; /* Cancel ongoing resize operations. */ _CMM_STORE_SHARED(ht->in_progress_destroy, 1); - /* Wait for in-flight resize operations to complete */ - urcu_workqueue_flush_queued_work(cds_lfht_workqueue); + if (attr) { + *attr = ht->caller_resize_attr; + ht->caller_resize_attr = NULL; + } + /* + * Queue destroy work after prior queued resize + * operations. Given there are no concurrent writers + * accessing the hash table at this point, no resize + * operations can be queued after this destroy work. + */ + urcu_workqueue_queue_work(cds_lfht_workqueue, + &ht->destroy_work, do_auto_resize_destroy_cb); + return 0; } ret = cds_lfht_delete_bucket(ht); if (ret) return ret; free_split_items_count(ht); if (attr) - *attr = ht->resize_attr; + *attr = ht->caller_resize_attr; ret = pthread_mutex_destroy(&ht->resize_mutex); if (ret) ret = -EBUSY; - if (ht->flags & CDS_LFHT_AUTO_RESIZE) - cds_lfht_fini_worker(ht->flavor); poison_free(ht); return ret; } @@ -2179,23 +2244,19 @@ static void cds_lfht_init_worker(const struct rcu_flavor_struct *flavor) flavor->register_rculfhash_atfork(&cds_lfht_atfork); mutex_lock(&cds_lfht_fork_mutex); - if (cds_lfht_workqueue_user_count++) - goto end; - cds_lfht_workqueue = urcu_workqueue_create(0, -1, NULL, - NULL, cds_lfht_worker_init, NULL, NULL, NULL, NULL, NULL); -end: + if (!cds_lfht_workqueue) + cds_lfht_workqueue = urcu_workqueue_create(0, -1, NULL, + NULL, cds_lfht_worker_init, NULL, NULL, NULL, NULL, NULL); mutex_unlock(&cds_lfht_fork_mutex); } -static void cds_lfht_fini_worker(const struct rcu_flavor_struct *flavor) +static void cds_lfht_exit(void) { mutex_lock(&cds_lfht_fork_mutex); - if (--cds_lfht_workqueue_user_count) - goto end; - urcu_workqueue_destroy(cds_lfht_workqueue); - cds_lfht_workqueue = NULL; -end: + if (cds_lfht_workqueue) { + urcu_workqueue_flush_queued_work(cds_lfht_workqueue); + urcu_workqueue_destroy(cds_lfht_workqueue); + cds_lfht_workqueue = NULL; + } mutex_unlock(&cds_lfht_fork_mutex); - - flavor->unregister_rculfhash_atfork(&cds_lfht_atfork); } diff --git a/src/urcu-call-rcu-impl.h b/src/urcu-call-rcu-impl.h index 0c252c12..c264da5f 100644 --- a/src/urcu-call-rcu-impl.h +++ b/src/urcu-call-rcu-impl.h @@ -102,7 +102,6 @@ static pthread_mutex_t call_rcu_mutex = PTHREAD_MUTEX_INITIALIZER; static struct call_rcu_data *default_call_rcu_data; static struct urcu_atfork *registered_rculfhash_atfork; -static unsigned long registered_rculfhash_atfork_refcount; /* * If the sched_getcpu() and sysconf(_SC_NPROCESSORS_CONF) calls are @@ -1018,20 +1017,20 @@ void call_rcu_after_fork_child(void) void urcu_register_rculfhash_atfork(struct urcu_atfork *atfork) { + if (CMM_LOAD_SHARED(registered_rculfhash_atfork)) + return; call_rcu_lock(&call_rcu_mutex); - if (registered_rculfhash_atfork_refcount++) - goto end; - registered_rculfhash_atfork = atfork; -end: + if (!registered_rculfhash_atfork) + registered_rculfhash_atfork = atfork; call_rcu_unlock(&call_rcu_mutex); } +/* + * This unregistration function is deprecated, meant only for internal + * use by rculfhash. + */ +__attribute__((noreturn)) void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork __attribute__((unused))) { - call_rcu_lock(&call_rcu_mutex); - if (--registered_rculfhash_atfork_refcount) - goto end; - registered_rculfhash_atfork = NULL; -end: - call_rcu_unlock(&call_rcu_mutex); + urcu_die(EPERM); }