From cbe1a1ab9ab454ae52f79c9a97aa3741195f156b Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 26 Aug 2024 15:26:16 -0400 Subject: [PATCH] Initial implementation of Mark & Sweep --- gc.c | 64 ++++- gc/default.c | 4 +- gc/gc.h | 7 + gc/gc_impl.h | 2 +- gc/mmtk.c | 533 ++++++++++++++++++++++++++++++++++---- gc/mmtk.h | 38 ++- gc/mmtk/Cargo.lock | 8 +- gc/mmtk/Cargo.toml | 6 +- gc/mmtk/src/abi.rs | 46 ++-- gc/mmtk/src/api.rs | 61 ++++- gc/mmtk/src/collection.rs | 6 +- gc/mmtk/src/scanning.rs | 37 +-- gc/mmtk/src/weak_proc.rs | 256 +++++++++--------- internal/gc.h | 2 +- ractor.c | 4 +- variable.c | 6 + 16 files changed, 806 insertions(+), 274 deletions(-) diff --git a/gc.c b/gc.c index 916b654e12a4e3..aeb0b5f24108b7 100644 --- a/gc.c +++ b/gc.c @@ -190,6 +190,12 @@ rb_gc_get_objspace(void) return GET_VM()->objspace; } +void * +rb_gc_get_ractor_newobj_cache(void) +{ + return GET_RACTOR()->newobj_cache; +} + void rb_gc_ractor_newobj_cache_foreach(void (*func)(void *cache, void *data), void *data) { @@ -324,6 +330,41 @@ rb_gc_rebuild_shape(VALUE obj, size_t size_pool_id) return (uint32_t)rb_shape_id(new_shape); } +struct st_table *generic_ivtbl_get(void); + +struct st_table * +rb_gc_get_generic_ivar_table(void) +{ + return generic_ivtbl_get(); +} + +struct st_table * +rb_gc_get_frozen_strings_table(void) +{ + return rb_vm_fstring_table(); +} + +extern rb_symbols_t ruby_global_symbols; +#define global_symbols ruby_global_symbols + +struct st_table * +rb_gc_get_global_symbols_table(void) +{ + return global_symbols.str_sym; +} + +struct st_table * +rb_gc_get_overloaded_cme_table(void) +{ + return GET_VM()->overloaded_cme_table; +} + +struct st_table * +rb_gc_get_ci_table(void) +{ + return GET_VM()->ci_table; +} + void rb_vm_update_references(void *ptr); #define rb_setjmp(env) RUBY_SETJMP(env) @@ -577,7 +618,7 @@ typedef struct gc_function_map { void *(*objspace_alloc)(void); void (*objspace_init)(void *objspace_ptr); void (*objspace_free)(void *objspace_ptr); - void *(*ractor_cache_alloc)(void *objspace_ptr); + void *(*ractor_cache_alloc)(void *objspace_ptr, void *ractor); void (*ractor_cache_free)(void *objspace_ptr, void *cache); void (*set_params)(void *objspace_ptr); void (*init)(void); @@ -2421,8 +2462,7 @@ mark_const_table_i(VALUE value, void *objspace) void rb_gc_mark_roots(void *objspace, const char **categoryp) { - rb_execution_context_t *ec = GET_EC(); - rb_vm_t *vm = rb_ec_vm_ptr(ec); + rb_vm_t *vm = GET_VM(); #define MARK_CHECKPOINT(category) do { \ if (categoryp) *categoryp = category; \ @@ -2432,9 +2472,6 @@ rb_gc_mark_roots(void *objspace, const char **categoryp) rb_vm_mark(vm); if (vm->self) rb_gc_impl_mark(objspace, vm->self); - MARK_CHECKPOINT("machine_context"); - mark_current_machine_context(objspace, ec); - MARK_CHECKPOINT("end_proc"); rb_mark_end_proc(); @@ -2451,6 +2488,17 @@ rb_gc_mark_roots(void *objspace, const char **categoryp) #endif MARK_CHECKPOINT("finish"); +} + +void +rb_gc_mark_thread_roots(void *objspace, void *ractor, const char **categoryp) +{ + if (ractor == NULL) ractor = GET_RACTOR(); + + rb_execution_context_t *ec = ((rb_ractor_t *)ractor)->threads.running_ec; + + MARK_CHECKPOINT("machine_context"); + mark_current_machine_context(objspace, ec); #undef MARK_CHECKPOINT } @@ -2736,9 +2784,9 @@ rb_obj_gc_flags(VALUE obj, ID* flags, size_t max) /* GC */ void * -rb_gc_ractor_cache_alloc(void) +rb_gc_ractor_cache_alloc(rb_ractor_t *ractor) { - return rb_gc_impl_ractor_cache_alloc(rb_gc_get_objspace()); + return rb_gc_impl_ractor_cache_alloc(rb_gc_get_objspace(), ractor); } void diff --git a/gc/default.c b/gc/default.c index e40c27c6cff74d..d8872df03a7694 100644 --- a/gc/default.c +++ b/gc/default.c @@ -4839,6 +4839,8 @@ mark_roots(rb_objspace_t *objspace, const char **categoryp) if (stress_to_class) rb_gc_mark(stress_to_class); rb_gc_mark_roots(objspace, categoryp); + + rb_gc_mark_thread_roots(objspace, NULL, categoryp); } static inline void @@ -6450,7 +6452,7 @@ rb_gc_impl_obj_flags(void *objspace_ptr, VALUE obj, ID* flags, size_t max) } void * -rb_gc_impl_ractor_cache_alloc(void *objspace_ptr) +rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor) { rb_objspace_t *objspace = objspace_ptr; diff --git a/gc/gc.h b/gc/gc.h index 2ee96365f2ebc2..99ab733086a453 100644 --- a/gc/gc.h +++ b/gc/gc.h @@ -26,12 +26,14 @@ void rb_gc_update_vm_references(void *objspace); void rb_gc_reachable_objects_from_callback(VALUE obj); void rb_gc_event_hook(VALUE obj, rb_event_flag_t event); void *rb_gc_get_objspace(void); +void *rb_gc_get_ractor_newobj_cache(void); size_t rb_size_mul_or_raise(size_t x, size_t y, VALUE exc); void rb_gc_run_obj_finalizer(VALUE objid, long count, VALUE (*callback)(long i, void *data), void *data); void rb_gc_set_pending_interrupt(void); void rb_gc_unset_pending_interrupt(void); bool rb_gc_obj_free(void *objspace, VALUE obj); void rb_gc_mark_roots(void *objspace, const char **categoryp); +void rb_gc_mark_thread_roots(void *objspace, void *ractor, const char **categoryp); void rb_gc_ractor_newobj_cache_foreach(void (*func)(void *cache, void *data), void *data); bool rb_gc_multi_ractor_p(void); void rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *passing_data); @@ -43,6 +45,11 @@ uint32_t rb_gc_get_shape(VALUE obj); void rb_gc_set_shape(VALUE obj, uint32_t shape_id); uint32_t rb_gc_rebuild_shape(VALUE obj, size_t size_pool_id); size_t rb_obj_memsize_of(VALUE obj); +struct st_table *rb_gc_get_generic_ivar_table(void); +struct st_table *rb_gc_get_frozen_strings_table(void); +struct st_table *rb_gc_get_global_symbols_table(void); +struct st_table *rb_gc_get_overloaded_cme_table(void); +struct st_table *rb_gc_get_ci_table(void); RUBY_SYMBOL_EXPORT_END void rb_ractor_finish_marking(void); diff --git a/gc/gc_impl.h b/gc/gc_impl.h index 12527845f7a2f9..4fb5ffe08207b6 100644 --- a/gc/gc_impl.h +++ b/gc/gc_impl.h @@ -28,7 +28,7 @@ GC_IMPL_FN void *rb_gc_impl_objspace_alloc(void); GC_IMPL_FN void rb_gc_impl_objspace_init(void *objspace_ptr); GC_IMPL_FN void rb_gc_impl_objspace_free(void *objspace_ptr); -GC_IMPL_FN void *rb_gc_impl_ractor_cache_alloc(void *objspace_ptr); +GC_IMPL_FN void *rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor); GC_IMPL_FN void rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache); GC_IMPL_FN void rb_gc_impl_set_params(void *objspace_ptr); GC_IMPL_FN void rb_gc_impl_init(void); diff --git a/gc/mmtk.c b/gc/mmtk.c index b8ad518cd06b4d..51e5bcd20ec2cf 100644 --- a/gc/mmtk.c +++ b/gc/mmtk.c @@ -1,67 +1,412 @@ // clang -I.. -L mmtk/target/debug -lmmtk_ruby -undefined dynamic_lookup -g -O3 -dynamiclib -o ../build/libgc.mmtk.dylib mmtk.c +#include #include #include "ruby/assert.h" +#include "ruby/atomic.h" +#include "ruby/debug.h" #include "gc/gc.h" #include "gc/gc_impl.h" #include "gc/mmtk.h" +struct MMTk_final_job { + struct MMTk_final_job *next; + enum { + MMTK_FINAL_JOB_DFREE, + MMTK_FINAL_JOB_FINALIZE, + } kind; + union { + struct { + void (*dfree)(void *); + void *data; + } dfree; + struct { + VALUE object_id; + VALUE finalizer_array; + } finalize; + } as; +}; + struct objspace { st_table *id_to_obj_tbl; st_table *obj_to_id_tbl; unsigned long long next_object_id; st_table *finalizer_table; + struct MMTk_final_job *finalizer_jobs; + rb_postponed_job_handle_t finalizer_postponed_job; + + unsigned long live_ractor_cache_count; + + int lock_lev; + + pthread_mutex_t mutex; + pthread_cond_t cond_world_stopped; + pthread_cond_t cond_world_started; + size_t stopped_ractors; + size_t start_the_world_count; }; -bool +#ifdef RB_THREAD_LOCAL_SPECIFIER +RB_THREAD_LOCAL_SPECIFIER struct MMTk_GCThreadTLS *rb_mmtk_gc_thread_tls; +#else +# error We currently need language-supported TLS +#endif + +#include + +static void +rb_mmtk_init_gc_worker_thread(MMTk_VMWorkerThread gc_thread_tls) +{ + rb_mmtk_gc_thread_tls = gc_thread_tls; +} + +static bool rb_mmtk_is_mutator(void) { return ruby_native_thread_p(); } +static void +rb_mmtk_stop_the_world(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + RUBY_ASSERT(objspace->lock_lev == 0); + objspace->lock_lev = rb_gc_vm_lock(); + + int err; + if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err)); + } + + while (objspace->stopped_ractors < 1) { + pthread_cond_wait(&objspace->cond_world_stopped, &objspace->mutex); + } + + if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err)); + } +} + +static void +rb_mmtk_resume_mutators(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + rb_gc_vm_unlock(objspace->lock_lev); + objspace->lock_lev = 0; + + int err; + if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err)); + } + + objspace->start_the_world_count++; + pthread_cond_broadcast(&objspace->cond_world_started); + + if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err)); + } +} + +static void +rb_mmtk_block_for_gc(MMTk_VMMutatorThread tls) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + int err; + if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err)); + } + + // Increment the stopped ractor count + objspace->stopped_ractors++; + if (objspace->stopped_ractors == 1) { + pthread_cond_broadcast(&objspace->cond_world_stopped); + } + + // Wait for GC end + size_t my_count = objspace->start_the_world_count; + + while (objspace->start_the_world_count < my_count + 1) { + pthread_cond_wait(&objspace->cond_world_started, &objspace->mutex); + } + + // Decrement the stopped ractor count + objspace->stopped_ractors--; + + if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err)); + } +} + +static size_t +rb_mmtk_number_of_mutators(void) +{ + // TODO + return 1; +} + +static void *mutator; + +static void +rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), void *data) +{ + // TODO + visit_mutator(mutator, data); +} + +static void +rb_mmtk_scan_gc_roots(void) +{ + rb_gc_mark_roots(rb_gc_get_objspace(), NULL); +} + +static int +pin_value(st_data_t key, st_data_t value, st_data_t data) +{ + rb_gc_impl_mark_and_pin((void *)data, (VALUE)value); + + return ST_CONTINUE; +} + +static void +rb_mmtk_scan_objspace(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + if (objspace->finalizer_table != NULL) { + st_foreach(objspace->finalizer_table, pin_value, (st_data_t)objspace); + } + + st_foreach(objspace->obj_to_id_tbl, gc_mark_tbl_no_pin_i, (st_data_t)objspace); +} + +static void +rb_mmtk_scan_roots_in_mutator_thread(MMTk_VMMutatorThread mutator, MMTk_VMWorkerThread worker) +{ + void *ractor = mutator; + + rb_gc_mark_thread_roots(rb_gc_get_objspace(), ractor, NULL); +} + +static void +rb_mmtk_scan_object_ruby_style(MMTk_ObjectReference object) +{ + rb_gc_mark_children(rb_gc_get_objspace(), (VALUE)object); +} + +static void +rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object) +{ + rb_gc_mark_children(rb_gc_get_objspace(), (VALUE)object); +} + +static void +rb_mmtk_call_obj_free(MMTk_ObjectReference object) +{ + VALUE obj = (VALUE)object; + + rb_gc_obj_free(rb_gc_get_objspace(), obj); +} + +static int +rb_mmtk_cleanup_generic_iv_tbl_i(st_data_t key, st_data_t val, st_data_t data) +{ + // TODO: make this more performant. We should just free val and return ST_DELETE. + MMTk_ObjectReference key_objref = (MMTk_ObjectReference)key; + if (!mmtk_is_live_object(key_objref)) { + rb_free_generic_ivar((VALUE)key); + + RB_FL_UNSET((VALUE)key, FL_EXIVAR); + } + + return ST_CONTINUE; +} + +static void +rb_mmtk_cleanup_generic_iv_tbl(void) +{ + struct st_table *table = rb_gc_get_generic_ivar_table(); + + st_foreach(table, rb_mmtk_cleanup_generic_iv_tbl_i, (st_data_t)0); +} + static size_t rb_mmtk_vm_live_bytes(void) { return 0; } +static int +rb_mmtk_update_frozen_strings_table_i(st_data_t key, st_data_t val, st_data_t data) +{ + RUBY_ASSERT(key == val); + RUBY_ASSERT(RB_BUILTIN_TYPE(key) == T_STRING); + RUBY_ASSERT(RB_FL_TEST(key, RSTRING_FSTR)); + + if (!mmtk_is_reachable((MMTk_ObjectReference)key)) { + RB_FL_UNSET(key, RSTRING_FSTR); + return ST_DELETE; + } + + return ST_CONTINUE; +} + +static void +rb_mmtk_update_frozen_strings_table(void) +{ + // TODO: replace with st_foreach_with_replace when GC is moving + st_foreach(rb_gc_get_frozen_strings_table(), rb_mmtk_update_frozen_strings_table_i, 0); +} + +static int +rb_mmtk_update_finalizer_table_i(st_data_t key, st_data_t value, st_data_t data) +{ + RUBY_ASSERT(RB_FL_TEST(key, RUBY_FL_FINALIZE)); + RUBY_ASSERT(mmtk_is_reachable((MMTk_ObjectReference)value)); + RUBY_ASSERT(RB_BUILTIN_TYPE(value) == T_ARRAY); + + struct objspace *objspace = (struct objspace *)data; + + if (!mmtk_is_reachable((MMTk_ObjectReference)key)) { + RB_FL_UNSET(key, RUBY_FL_FINALIZE); + + struct MMTk_final_job *job = xmalloc(sizeof(struct MMTk_final_job)); + job->next = objspace->finalizer_jobs; + job->kind = MMTK_FINAL_JOB_FINALIZE; + job->as.finalize.object_id = rb_obj_id((VALUE)key); + job->as.finalize.finalizer_array = (VALUE)value; + + objspace->finalizer_jobs = job; + + // TODO: maybe only call this once + rb_postponed_job_trigger(objspace->finalizer_postponed_job); + + return ST_DELETE; + } + + return ST_CONTINUE; +} + +static void +rb_mmtk_update_finalizer_table(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + // TODO: replace with st_foreach_with_replace when GC is moving + st_foreach(objspace->finalizer_table, rb_mmtk_update_finalizer_table_i, (st_data_t)objspace); +} + +static int +rb_mmtk_update_obj_id_tables_i(st_data_t key, st_data_t val, st_data_t data) +{ + RUBY_ASSERT(RB_FL_TEST(key, FL_SEEN_OBJ_ID)); + + // TODO: is it safe to delete if key has finalizer? + if (!mmtk_is_reachable((MMTk_ObjectReference)key)) { + RB_FL_UNSET(key, FL_SEEN_OBJ_ID); + return ST_DELETE; + } + + return ST_CONTINUE; +} + +static void +rb_mmtk_update_obj_id_tables(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + st_foreach(objspace->obj_to_id_tbl, rb_mmtk_update_obj_id_tables_i, 0); +} + +static int +rb_mmtk_update_global_symbols_table_i(st_data_t key, st_data_t val, st_data_t data) +{ + RUBY_ASSERT(RB_BUILTIN_TYPE(key) == T_SYMBOL); + + if (!mmtk_is_reachable((MMTk_ObjectReference)key)) { + return ST_DELETE; + } + + return ST_CONTINUE; +} + +static void +rb_mmtk_update_global_symbols_table(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + // st_foreach(rb_gc_get_global_symbols_table(), rb_mmtk_update_global_symbols_table_i, 0); +} + +static int +rb_mmtk_update_overloaded_cme_table_i(st_data_t key, st_data_t val, st_data_t data) +{ + RUBY_ASSERT(RB_BUILTIN_TYPE(key) == T_SYMBOL); + + if (!mmtk_is_reachable((MMTk_ObjectReference)key)) { + return ST_DELETE; + } + + return ST_CONTINUE; +} + +static void +rb_mmtk_update_overloaded_cme_table(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + st_foreach(rb_gc_get_overloaded_cme_table(), rb_mmtk_update_overloaded_cme_table_i, 0); +} + +static int +rb_mmtk_update_ci_table_i(st_data_t key, st_data_t val, st_data_t data) +{ + if (!mmtk_is_reachable((MMTk_ObjectReference)key)) { + return ST_DELETE; + } + + return ST_CONTINUE; +} + +static void +rb_mmtk_update_ci_table(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + st_foreach(rb_gc_get_ci_table(), rb_mmtk_update_ci_table_i, 0); +} + // Bootup MMTk_RubyUpcalls ruby_upcalls = { - NULL, + rb_mmtk_init_gc_worker_thread, NULL, rb_mmtk_is_mutator, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + rb_mmtk_stop_the_world, + rb_mmtk_resume_mutators, + rb_mmtk_block_for_gc, + rb_mmtk_number_of_mutators, + rb_mmtk_get_mutators, + rb_mmtk_scan_gc_roots, + rb_mmtk_scan_objspace, + rb_mmtk_scan_roots_in_mutator_thread, + rb_mmtk_scan_object_ruby_style, + rb_mmtk_call_gc_mark_children, + rb_mmtk_call_obj_free, + rb_mmtk_cleanup_generic_iv_tbl, NULL, NULL, rb_mmtk_vm_live_bytes, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + rb_mmtk_update_frozen_strings_table, + rb_mmtk_update_finalizer_table, + rb_mmtk_update_obj_id_tables, + rb_mmtk_update_global_symbols_table, + rb_mmtk_update_overloaded_cme_table, + rb_mmtk_update_ci_table, NULL, NULL, NULL, @@ -80,6 +425,7 @@ rb_gc_impl_objspace_alloc(void) } static void objspace_obj_id_init(struct objspace *objspace); +static void gc_run_finalizers(void *data); void rb_gc_impl_objspace_init(void *objspace_ptr) @@ -89,6 +435,11 @@ rb_gc_impl_objspace_init(void *objspace_ptr) objspace_obj_id_init(objspace); objspace->finalizer_table = st_init_numtable(); + objspace->finalizer_postponed_job = rb_postponed_job_preregister(0, gc_run_finalizers, objspace); + + objspace->mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; + objspace->cond_world_stopped = (pthread_cond_t)PTHREAD_COND_INITIALIZER; + objspace->cond_world_started = (pthread_cond_t)PTHREAD_COND_INITIALIZER; } void @@ -98,16 +449,28 @@ rb_gc_impl_objspace_free(void *objspace_ptr) } void * -rb_gc_impl_ractor_cache_alloc(void *objspace_ptr) +rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor) { - // TODO: pass not NULL to tls - return mmtk_bind_mutator(NULL); + struct objspace *objspace = objspace_ptr; + if (objspace->live_ractor_cache_count == 0) { + mmtk_initialize_collection(ractor); + } + objspace->live_ractor_cache_count++; + + mutator = mmtk_bind_mutator(ractor); + return mutator; + + // return mmtk_bind_mutator(ractor); } void rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache) { // TODO: implement mmtk_destroy_mutator + struct objspace *objspace = objspace_ptr; + + RUBY_ASSERT(objspace->live_ractor_cache_count > 1); + objspace->live_ractor_cache_count--; } void rb_gc_impl_set_params(void *objspace_ptr) { } @@ -220,9 +583,8 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags mmtk_post_alloc(cache_ptr, (void*)alloc_obj, alloc_size + 8, MMTK_ALLOCATION_SEMANTICS_DEFAULT); - if (rb_gc_shutdown_call_finalizer_p((VALUE)alloc_obj)) { - mmtk_add_obj_free_candidate(alloc_obj); - } + // TODO: only add when object needs obj_free to be called + mmtk_add_obj_free_candidate(alloc_obj); return (VALUE)alloc_obj; } @@ -280,28 +642,42 @@ rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size) } void rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff) { } + // Marking void rb_gc_impl_mark(void *objspace_ptr, VALUE obj) { - rb_bug("unimplemented"); + if (RB_SPECIAL_CONST_P(obj)) return; + + rb_mmtk_gc_thread_tls->object_closure.c_function(rb_mmtk_gc_thread_tls->object_closure.rust_closure, + rb_mmtk_gc_thread_tls->gc_context, + (MMTk_ObjectReference)obj, + false); } void rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr) { - rb_bug("unimplemented"); + if (RB_SPECIAL_CONST_P(*ptr)) return; + + // TODO: make it movable + rb_gc_impl_mark(objspace_ptr, *ptr); } void rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj) { - rb_bug("unimplemented"); + if (RB_SPECIAL_CONST_P(obj)) return; + + // TODO: also pin + rb_gc_impl_mark(objspace_ptr, obj); } void rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj) { - rb_bug("unimplemented"); + if (rb_gc_impl_pointer_to_heap_p(objspace_ptr, (const void *)obj)) { + rb_gc_impl_mark_and_pin(objspace_ptr, obj); + } } void @@ -315,12 +691,6 @@ rb_gc_impl_remove_weak(void *objspace_ptr, VALUE parent_obj, VALUE *ptr) rb_bug("unimplemented"); } -void -rb_gc_impl_objspace_mark(void *objspace_ptr) -{ - rb_bug("unimplemented"); -} - // Compaction bool rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj) @@ -333,10 +703,26 @@ rb_gc_impl_location(void *objspace_ptr, VALUE value) { rb_bug("unimplemented"); } + // Write barriers -void rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b) { } -void rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj) { } -void rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj) { } +void +rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b) +{ + mmtk_object_reference_write_post(rb_gc_get_ractor_newobj_cache(), (MMTk_ObjectReference)a); +} + +void +rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj) +{ + mmtk_register_wb_unprotected_object((MMTk_ObjectReference)obj); +} + +void +rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj) +{ + mmtk_object_reference_write_post(rb_gc_get_ractor_newobj_cache(), (MMTk_ObjectReference)obj); +} + // Heap walking struct each_objects_data { bool stop; @@ -371,7 +757,50 @@ rb_gc_impl_each_objects(void *objspace_ptr, int (*callback)(void *, void *, size } void rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data) { } + // Finalizers +static VALUE +gc_run_finalizers_get_final(long i, void *data) +{ + VALUE table = (VALUE)data; + + return RARRAY_AREF(table, i); +} + +static void +gc_run_finalizers(void *data) +{ + struct objspace *objspace = data; + + rb_gc_set_pending_interrupt(); + + while (true) { + struct MMTk_final_job *jobs = RUBY_ATOMIC_PTR_EXCHANGE(objspace->finalizer_jobs, NULL); + if (jobs == NULL) break; + + while (jobs != NULL) { + struct MMTk_final_job *job = jobs; + jobs = job->next; + + switch (job->kind) { + case MMTK_FINAL_JOB_DFREE: + // TODO + break; + case MMTK_FINAL_JOB_FINALIZE: + rb_gc_run_obj_finalizer( + job->as.finalize.object_id, + RARRAY_LEN(job->as.finalize.finalizer_array), + gc_run_finalizers_get_final, + (void *)job->as.finalize.finalizer_array + ); + break; + } + } + } + + rb_gc_unset_pending_interrupt(); +} + void rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data) { @@ -591,9 +1020,17 @@ size_t rb_gc_impl_gc_count(void *objspace_ptr) { } VALUE rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE key) { } size_t rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) { } size_t rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym) { } + // Miscellaneous size_t rb_gc_impl_obj_flags(void *objspace_ptr, VALUE obj, ID* flags, size_t max) { } -bool rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr) { } + +bool +rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr) +{ + if (ptr == NULL) return false; + if ((uintptr_t)ptr % sizeof(void*) != 0) return false; + return mmtk_is_mmtk_object((MMTk_Address)ptr); +} bool rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE obj) diff --git a/gc/mmtk.h b/gc/mmtk.h index b317bda4631859..f1509eda5c1f08 100644 --- a/gc/mmtk.h +++ b/gc/mmtk.h @@ -59,18 +59,13 @@ typedef struct MMTk_RubyUpcalls { void (*init_gc_worker_thread)(struct MMTk_GCThreadTLS *gc_worker_tls); struct MMTk_GCThreadTLS *(*get_gc_thread_tls)(void); bool (*is_mutator)(void); - void (*stop_the_world)(MMTk_VMWorkerThread tls); - void (*resume_mutators)(MMTk_VMWorkerThread tls); + void (*stop_the_world)(void); + void (*resume_mutators)(void); void (*block_for_gc)(MMTk_VMMutatorThread tls); size_t (*number_of_mutators)(void); void (*get_mutators)(void (*visit_mutator)(MMTk_Mutator*, void*), void *data); - void (*scan_vm_roots)(void); - void (*scan_finalizer_tbl_roots)(void); - void (*scan_end_proc_roots)(void); - void (*scan_global_tbl_roots)(void); - void (*scan_obj_to_id_tbl_roots)(void); - void (*scan_misc_roots)(void); - void (*scan_final_jobs_roots)(void); + void (*scan_gc_roots)(void); + void (*scan_objspace)(void); void (*scan_roots_in_mutator_thread)(MMTk_VMMutatorThread mutator_tls, MMTk_VMWorkerThread worker_tls); void (*scan_object_ruby_style)(MMTk_ObjectReference object); @@ -92,17 +87,6 @@ typedef struct MMTk_RubyUpcalls { struct MMTk_st_table *(*get_global_symbols_table)(void); struct MMTk_st_table *(*get_overloaded_cme_table)(void); struct MMTk_st_table *(*get_ci_table)(void); - void (*st_get_size_info)(const struct MMTk_st_table *table, - size_t *entries_start, - size_t *entries_bound, - size_t *bins_num); - void (*st_update_entries_range)(struct MMTk_st_table *table, - size_t begin, - size_t end, - bool weak_keys, - bool weak_records, - bool forward); - void (*st_update_bins_range)(struct MMTk_st_table *table, size_t begin, size_t end); } MMTk_RubyUpcalls; typedef struct MMTk_RawVecOfObjRef { @@ -111,14 +95,22 @@ typedef struct MMTk_RawVecOfObjRef { size_t capa; } MMTk_RawVecOfObjRef; +bool mmtk_is_live_object(MMTk_ObjectReference object); + +bool mmtk_is_reachable(MMTk_ObjectReference object); + MMTk_Builder *mmtk_builder_default(void); void mmtk_init_binding(MMTk_Builder *builder, const struct MMTk_RubyBindingOptions *_binding_options, const struct MMTk_RubyUpcalls *upcalls); +void mmtk_initialize_collection(MMTk_VMThread tls); + MMTk_Mutator *mmtk_bind_mutator(MMTk_VMMutatorThread tls); +void mmtk_handle_user_collection_request(MMTk_VMMutatorThread tls); + MMTk_Address mmtk_alloc(MMTk_Mutator *mutator, size_t size, size_t align, @@ -132,10 +124,16 @@ void mmtk_post_alloc(MMTk_Mutator *mutator, void mmtk_add_obj_free_candidate(MMTk_ObjectReference object); +void mmtk_object_reference_write_post(MMTk_Mutator *mutator, MMTk_ObjectReference object); + +void mmtk_register_wb_unprotected_object(MMTk_ObjectReference object); + void mmtk_enumerate_objects(void (*callback)(MMTk_ObjectReference, void*), void *data); struct MMTk_RawVecOfObjRef mmtk_get_all_obj_free_candidates(void); void mmtk_free_raw_vec_of_obj_ref(struct MMTk_RawVecOfObjRef raw_vec); +bool mmtk_is_mmtk_object(MMTk_Address addr); + #endif /* MMTK_H */ diff --git a/gc/mmtk/Cargo.lock b/gc/mmtk/Cargo.lock index ea0e528efbd6ce..854eca8a66e2e1 100644 --- a/gc/mmtk/Cargo.lock +++ b/gc/mmtk/Cargo.lock @@ -431,8 +431,8 @@ dependencies = [ [[package]] name = "mmtk" -version = "0.26.0" -source = "git+https://github.com/wks/mmtk-core.git?rev=7d3f79d4e50dacec881252562c8c7946e2513e55#7d3f79d4e50dacec881252562c8c7946e2513e55" +version = "0.27.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=160b7702fccda133c9407234821ad35103623179#160b7702fccda133c9407234821ad35103623179" dependencies = [ "atomic", "atomic-traits", @@ -467,8 +467,8 @@ dependencies = [ [[package]] name = "mmtk-macros" -version = "0.26.0" -source = "git+https://github.com/wks/mmtk-core.git?rev=7d3f79d4e50dacec881252562c8c7946e2513e55#7d3f79d4e50dacec881252562c8c7946e2513e55" +version = "0.27.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=160b7702fccda133c9407234821ad35103623179#160b7702fccda133c9407234821ad35103623179" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/gc/mmtk/Cargo.toml b/gc/mmtk/Cargo.toml index c5c7b9987a3dbb..e2684875c550e2 100644 --- a/gc/mmtk/Cargo.toml +++ b/gc/mmtk/Cargo.toml @@ -29,11 +29,11 @@ probe = "0.5" features = ["is_mmtk_object", "object_pinning", "sticky_immix_non_moving_nursery"] # Uncomment the following lines to use mmtk-core from the official repository. -git = "https://github.com/wks/mmtk-core.git" -rev = "7d3f79d4e50dacec881252562c8c7946e2513e55" +git = "https://github.com/mmtk/mmtk-core.git" +rev = "160b7702fccda133c9407234821ad35103623179" # Uncomment the following line to use mmtk-core from a local repository. -# path = "../../mmtk-core" +# path = "../../../mmtk/mmtk-core" [features] default = [] diff --git a/gc/mmtk/src/abi.rs b/gc/mmtk/src/abi.rs index b27c01454508c4..00dcb84fbba69c 100644 --- a/gc/mmtk/src/abi.rs +++ b/gc/mmtk/src/abi.rs @@ -347,21 +347,16 @@ pub struct RubyUpcalls { pub init_gc_worker_thread: extern "C" fn(gc_worker_tls: *mut GCThreadTLS), pub get_gc_thread_tls: extern "C" fn() -> *mut GCThreadTLS, pub is_mutator: extern "C" fn() -> bool, - pub stop_the_world: extern "C" fn(tls: VMWorkerThread), - pub resume_mutators: extern "C" fn(tls: VMWorkerThread), + pub stop_the_world: extern "C" fn(), + pub resume_mutators: extern "C" fn(), pub block_for_gc: extern "C" fn(tls: VMMutatorThread), pub number_of_mutators: extern "C" fn() -> usize, pub get_mutators: extern "C" fn( visit_mutator: extern "C" fn(*mut RubyMutator, *mut libc::c_void), data: *mut libc::c_void, ), - pub scan_vm_roots: extern "C" fn(), - pub scan_finalizer_tbl_roots: extern "C" fn(), - pub scan_end_proc_roots: extern "C" fn(), - pub scan_global_tbl_roots: extern "C" fn(), - pub scan_obj_to_id_tbl_roots: extern "C" fn(), - pub scan_misc_roots: extern "C" fn(), - pub scan_final_jobs_roots: extern "C" fn(), + pub scan_gc_roots: extern "C" fn(), + pub scan_objspace: extern "C" fn(), pub scan_roots_in_mutator_thread: extern "C" fn(mutator_tls: VMMutatorThread, worker_tls: VMWorkerThread), pub scan_object_ruby_style: extern "C" fn(object: ObjectReference), @@ -383,22 +378,23 @@ pub struct RubyUpcalls { pub get_global_symbols_table: extern "C" fn() -> *mut st_table, pub get_overloaded_cme_table: extern "C" fn() -> *mut st_table, pub get_ci_table: extern "C" fn() -> *mut st_table, - pub st_get_size_info: extern "C" fn( - table: *const st_table, - entries_start: *mut libc::size_t, - entries_bound: *mut libc::size_t, - bins_num: *mut libc::size_t, - ), - pub st_update_entries_range: extern "C" fn( - table: *mut st_table, - begin: libc::size_t, - end: libc::size_t, - weak_keys: bool, - weak_records: bool, - forward: bool, - ), - pub st_update_bins_range: - extern "C" fn(table: *mut st_table, begin: libc::size_t, end: libc::size_t), + + // pub st_get_size_info: extern "C" fn( + // table: *const st_table, + // entries_start: *mut libc::size_t, + // entries_bound: *mut libc::size_t, + // bins_num: *mut libc::size_t, + // ), + // pub st_update_entries_range: extern "C" fn( + // table: *mut st_table, + // begin: libc::size_t, + // end: libc::size_t, + // weak_keys: bool, + // weak_records: bool, + // forward: bool, + // ), + // pub st_update_bins_range: + // extern "C" fn(table: *mut st_table, begin: libc::size_t, end: libc::size_t), } unsafe impl Sync for RubyUpcalls {} diff --git a/gc/mmtk/src/api.rs b/gc/mmtk/src/api.rs index 6a8c116498453f..ed74e8ebdc2ea1 100644 --- a/gc/mmtk/src/api.rs +++ b/gc/mmtk/src/api.rs @@ -5,19 +5,32 @@ use crate::binding; use crate::binding::RubyBinding; use crate::mmtk; use crate::Ruby; +use crate::RubySlot; use mmtk::memory_manager; use mmtk::memory_manager::mmtk_init; use mmtk::util::constants::MIN_OBJECT_SIZE; +use mmtk::util::options::GCTriggerSelector; use mmtk::util::options::PlanSelector; use mmtk::util::Address; use mmtk::util::ObjectReference; use mmtk::util::VMMutatorThread; +use mmtk::util::VMThread; use mmtk::AllocationSemantics; use mmtk::MMTKBuilder; use mmtk::Mutator; pub type RubyMutator = Mutator; +#[no_mangle] +pub extern "C" fn mmtk_is_live_object(object: ObjectReference) -> bool { + memory_manager::is_live_object::(object) +} + +#[no_mangle] +pub extern "C" fn mmtk_is_reachable(object: ObjectReference) -> bool { + object.is_reachable::() +} + // =============== Bootup =============== #[no_mangle] @@ -26,9 +39,12 @@ pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder { builder.options.no_finalizer.set(true); // Hard code NoGC for now - let plan_selector = "NoGC".parse::().unwrap(); + let plan_selector = "MarkSweep".parse::().unwrap(); builder.options.plan.set(plan_selector); + // 100MiB + builder.options.gc_trigger.set(GCTriggerSelector::FixedHeapSize(100 << 20)); + Box::into_raw(Box::new(builder)) } @@ -52,11 +68,23 @@ pub extern "C" fn mmtk_init_binding( .unwrap_or_else(|_| panic!("Binding is already initialized")); } +#[no_mangle] +pub extern "C" fn mmtk_initialize_collection(tls: VMThread) { + memory_manager::initialize_collection(mmtk(), tls) +} + #[no_mangle] pub extern "C" fn mmtk_bind_mutator(tls: VMMutatorThread) -> *mut RubyMutator { Box::into_raw(memory_manager::bind_mutator(mmtk(), tls)) } +// =============== GC =============== + +#[no_mangle] +pub extern "C" fn mmtk_handle_user_collection_request(tls: VMMutatorThread) { + memory_manager::handle_user_collection_request::(mmtk(), tls); +} + // =============== Object allocation =============== #[no_mangle] @@ -93,6 +121,28 @@ pub extern "C" fn mmtk_add_obj_free_candidate(object: ObjectReference) { binding().weak_proc.add_obj_free_candidate(object) } +// =============== Write barriers =============== + +#[no_mangle] +pub extern "C" fn mmtk_object_reference_write_post( + mutator: *mut RubyMutator, + object: ObjectReference, +) { + let ignored_slot = RubySlot::from_address(Address::ZERO); + let ignored_target = ObjectReference::from_raw_address(Address::ZERO); + mmtk::memory_manager::object_reference_write_post( + unsafe { &mut *mutator }, + object, + ignored_slot, + ignored_target, + ) +} + +#[no_mangle] +pub extern "C" fn mmtk_register_wb_unprotected_object(object: ObjectReference) { + crate::binding().register_wb_unprotected_object(object) +} + // =============== Heap walking =============== #[no_mangle] @@ -117,3 +167,12 @@ pub extern "C" fn mmtk_get_all_obj_free_candidates() -> RawVecOfObjRef { pub extern "C" fn mmtk_free_raw_vec_of_obj_ref(raw_vec: RawVecOfObjRef) { unsafe { raw_vec.into_vec() }; } + +// =============== Miscellaneous =============== + +#[no_mangle] +pub extern "C" fn mmtk_is_mmtk_object(addr: Address) -> bool { + debug_assert!(!addr.is_zero()); + debug_assert!(addr.is_aligned_to(mmtk::util::is_mmtk_object::VO_BIT_REGION_SIZE)); + memory_manager::is_mmtk_object(addr).is_some() +} diff --git a/gc/mmtk/src/collection.rs b/gc/mmtk/src/collection.rs index ad4da8ba7edee6..45312a87673839 100644 --- a/gc/mmtk/src/collection.rs +++ b/gc/mmtk/src/collection.rs @@ -20,7 +20,7 @@ impl Collection for VMCollection { where F: FnMut(&'static mut mmtk::Mutator), { - (upcalls().stop_the_world)(tls); + (upcalls().stop_the_world)(); crate::binding().ppp_registry.pin_ppp_children(tls); (upcalls().get_mutators)( Self::notify_mutator_ready::, @@ -28,8 +28,8 @@ impl Collection for VMCollection { ); } - fn resume_mutators(tls: VMWorkerThread) { - (upcalls().resume_mutators)(tls); + fn resume_mutators(_tls: VMWorkerThread) { + (upcalls().resume_mutators)(); } fn block_for_gc(tls: VMMutatorThread) { diff --git a/gc/mmtk/src/scanning.rs b/gc/mmtk/src/scanning.rs index 3fb8ca1fb90bde..9bded9839e8a3b 100644 --- a/gc/mmtk/src/scanning.rs +++ b/gc/mmtk/src/scanning.rs @@ -78,13 +78,8 @@ impl Scanning for VMScanning { fn scan_vm_specific_roots(tls: VMWorkerThread, factory: impl RootsWorkFactory) { let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) }; let root_scanning_work_packets: Vec>> = vec![ - Box::new(ScanVMRoots::new(factory.clone())), - Box::new(ScanFinalizerTblRoots::new(factory.clone())), - Box::new(ScanEndProcRoots::new(factory.clone())), - Box::new(ScanGlobalTblRoots::new(factory.clone())), - Box::new(ScanObjToIdTblRoots::new(factory.clone())), - Box::new(ScanMiscRoots::new(factory.clone())), - Box::new(ScanFinalJobsRoots::new(factory.clone())), + Box::new(ScanGCRoots::new(factory.clone())), + Box::new(ScanObjspace::new(factory.clone())), ]; gc_tls.worker().scheduler().work_buckets[WorkBucketStage::Prepare] .bulk_add(root_scanning_work_packets); @@ -237,32 +232,12 @@ macro_rules! define_global_root_scanner { }; } -define_global_root_scanner!(ScanVMRoots, { - (crate::upcalls().scan_vm_roots)(); +define_global_root_scanner!(ScanGCRoots, { + (crate::upcalls().scan_gc_roots)(); }); -define_global_root_scanner!(ScanFinalizerTblRoots, { - (crate::upcalls().scan_finalizer_tbl_roots)(); -}); - -define_global_root_scanner!(ScanEndProcRoots, { - (crate::upcalls().scan_end_proc_roots)(); -}); - -define_global_root_scanner!(ScanGlobalTblRoots, { - (crate::upcalls().scan_global_tbl_roots)(); -}); - -define_global_root_scanner!(ScanObjToIdTblRoots, { - (crate::upcalls().scan_obj_to_id_tbl_roots)(); -}); - -define_global_root_scanner!(ScanMiscRoots, { - (crate::upcalls().scan_misc_roots)(); -}); - -define_global_root_scanner!(ScanFinalJobsRoots, { - (crate::upcalls().scan_final_jobs_roots)(); +define_global_root_scanner!(ScanObjspace, { + (crate::upcalls().scan_objspace)(); }); struct ScanWbUnprotectedRoots> { diff --git a/gc/mmtk/src/weak_proc.rs b/gc/mmtk/src/weak_proc.rs index f231a7ddf51fc2..da137c7af44c08 100644 --- a/gc/mmtk/src/weak_proc.rs +++ b/gc/mmtk/src/weak_proc.rs @@ -66,93 +66,93 @@ impl WeakProcessor { worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].bulk_add(vec![ Box::new(UpdateGenericIvTbl) as _, - // Box::new(UpdateFrozenStringsTable) as _, + Box::new(UpdateFrozenStringsTable) as _, Box::new(UpdateFinalizerTable) as _, Box::new(UpdateObjIdTables) as _, - // Box::new(UpdateGlobalSymbolsTable) as _, + Box::new(UpdateGlobalSymbolsTable) as _, Box::new(UpdateOverloadedCmeTable) as _, Box::new(UpdateCiTable) as _, Box::new(UpdateWbUnprotectedObjectsList) as _, ]); - let forward = crate::mmtk().get_plan().current_gc_may_move_object(); + // let forward = crate::mmtk().get_plan().current_gc_may_move_object(); // Experimenting with frozen strings table - Self::process_weak_table_chunked( - "frozen strings", - (upcalls().get_frozen_strings_table)(), - true, - false, - forward, - worker, - ); - - Self::process_weak_table_chunked( - "global symbols", - (upcalls().get_global_symbols_table)(), - false, - true, - forward, - worker, - ); + // Self::process_weak_table_chunked( + // "frozen strings", + // (upcalls().get_frozen_strings_table)(), + // true, + // false, + // forward, + // worker, + // ); + + // Self::process_weak_table_chunked( + // "global symbols", + // (upcalls().get_global_symbols_table)(), + // false, + // true, + // forward, + // worker, + // ); } - pub fn process_weak_table_chunked( - name: &str, - table: *mut st_table, - weak_keys: bool, - weak_values: bool, - forward: bool, - worker: &mut GCWorker, - ) { - let mut entries_start = 0; - let mut entries_bound = 0; - let mut bins_num = 0; - (upcalls().st_get_size_info)(table, &mut entries_start, &mut entries_bound, &mut bins_num); - debug!( - "name: {name}, entries_start: {entries_start}, entries_bound: {entries_bound}, bins_num: {bins_num}" - ); - - let entries_chunk_size = crate::binding().st_entries_chunk_size; - let bins_chunk_size = crate::binding().st_bins_chunk_size; - - let after_all = Arc::new(AfterAll::new(WorkBucketStage::VMRefClosure)); - - let entries_packets = (entries_start..entries_bound) - .step_by(entries_chunk_size) - .map(|begin| { - let end = (begin + entries_chunk_size).min(entries_bound); - let after_all = after_all.clone(); - Box::new(UpdateTableEntriesParallel { - name: name.to_string(), - table, - begin, - end, - weak_keys, - weak_values, - forward, - after_all, - }) as _ - }) - .collect::>(); - after_all.count_up(entries_packets.len()); - - let bins_packets = (0..bins_num) - .step_by(entries_chunk_size) - .map(|begin| { - let end = (begin + bins_chunk_size).min(bins_num); - Box::new(UpdateTableBinsParallel { - name: name.to_string(), - table, - begin, - end, - }) as _ - }) - .collect::>(); - after_all.add_packets(bins_packets); - - worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].bulk_add(entries_packets); - } + // pub fn process_weak_table_chunked( + // name: &str, + // table: *mut st_table, + // weak_keys: bool, + // weak_values: bool, + // forward: bool, + // worker: &mut GCWorker, + // ) { + // let mut entries_start = 0; + // let mut entries_bound = 0; + // let mut bins_num = 0; + // (upcalls().st_get_size_info)(table, &mut entries_start, &mut entries_bound, &mut bins_num); + // debug!( + // "name: {name}, entries_start: {entries_start}, entries_bound: {entries_bound}, bins_num: {bins_num}" + // ); + + // let entries_chunk_size = crate::binding().st_entries_chunk_size; + // let bins_chunk_size = crate::binding().st_bins_chunk_size; + + // let after_all = Arc::new(AfterAll::new(WorkBucketStage::VMRefClosure)); + + // let entries_packets = (entries_start..entries_bound) + // .step_by(entries_chunk_size) + // .map(|begin| { + // let end = (begin + entries_chunk_size).min(entries_bound); + // let after_all = after_all.clone(); + // Box::new(UpdateTableEntriesParallel { + // name: name.to_string(), + // table, + // begin, + // end, + // weak_keys, + // weak_values, + // forward, + // after_all, + // }) as _ + // }) + // .collect::>(); + // after_all.count_up(entries_packets.len()); + + // let bins_packets = (0..bins_num) + // .step_by(entries_chunk_size) + // .map(|begin| { + // let end = (begin + bins_chunk_size).min(bins_num); + // Box::new(UpdateTableBinsParallel { + // name: name.to_string(), + // table, + // begin, + // end, + // }) as _ + // }) + // .collect::>(); + // after_all.add_packets(bins_packets); + + // worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].bulk_add(entries_packets); + // } /// Update generic instance variable tables. /// @@ -287,57 +287,61 @@ define_global_table_processor!(UpdateOverloadedCmeTable, { (crate::upcalls().update_overloaded_cme_table)() }); -define_global_table_processor!(UpdateCiTable, (crate::upcalls().update_ci_table)()); - -struct UpdateTableEntriesParallel { - name: String, - table: *mut st_table, - begin: usize, - end: usize, - weak_keys: bool, - weak_values: bool, - forward: bool, - after_all: Arc, -} - -unsafe impl Send for UpdateTableEntriesParallel {} - -impl UpdateTableEntriesParallel {} - -impl GCWork for UpdateTableEntriesParallel { - fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static mmtk::MMTK) { - debug!("Updating entries of {} table", self.name); - (upcalls().st_update_entries_range)( - self.table, - self.begin, - self.end, - self.weak_keys, - self.weak_values, - self.forward, - ); - debug!("Done updating entries of {} table", self.name); - self.after_all.count_down(worker); - } -} - -struct UpdateTableBinsParallel { - name: String, - table: *mut st_table, - begin: usize, - end: usize, -} - -unsafe impl Send for UpdateTableBinsParallel {} - -impl UpdateTableBinsParallel {} +define_global_table_processor!(UpdateCiTable, { + (crate::upcalls().update_ci_table)() +}); -impl GCWork for UpdateTableBinsParallel { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static mmtk::MMTK) { - debug!("Updating bins of {} table", self.name); - (upcalls().st_update_bins_range)(self.table, self.begin, self.end); - debug!("Done updating bins of {} table", self.name); - } -} +// impl GCWork for UpdateFrozenStringsTable { +// fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static mmtk::MMTK) { + +// } +// } + +// struct UpdateSTTableEntries { +// name: String, +// table: *mut st_table, +// // begin: usize, +// // end: usize, +// weak_keys: bool, +// weak_values: bool, +// forward: bool, +// after_all: Arc, +// } + +// unsafe impl Send for UpdateSTTableEntries {} + +// impl GCWork for UpdateSTTableEntries { +// fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static mmtk::MMTK) { +// debug!("Updating entries of {} table", self.name); +// (upcalls().st_update_entries_range)( +// self.table, +// self.begin, +// self.end, +// self.weak_keys, +// self.weak_values, +// self.forward, +// ); +// debug!("Done updating entries of {} table", self.name); +// self.after_all.count_down(worker); +// } +// } + +// struct UpdateTableBinsParallel { +// name: String, +// table: *mut st_table, +// begin: usize, +// end: usize, +// } + +// unsafe impl Send for UpdateTableBinsParallel {} + +// impl GCWork for UpdateTableBinsParallel { +// fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static mmtk::MMTK) { +// debug!("Updating bins of {} table", self.name); +// (upcalls().st_update_bins_range)(self.table, self.begin, self.end); +// debug!("Done updating bins of {} table", self.name); +// } +// } struct UpdateWbUnprotectedObjectsList; diff --git a/internal/gc.h b/internal/gc.h index 81ca1a51d8b6f7..38f3d27ac65890 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -197,7 +197,7 @@ static inline void *ruby_sized_xrealloc_inlined(void *ptr, size_t new_size, size static inline void *ruby_sized_xrealloc2_inlined(void *ptr, size_t new_count, size_t elemsiz, size_t old_count) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2, 3)); static inline void ruby_sized_xfree_inlined(void *ptr, size_t size); -void *rb_gc_ractor_cache_alloc(void); +void *rb_gc_ractor_cache_alloc(rb_ractor_t *ractor); void rb_gc_ractor_cache_free(void *cache); bool rb_gc_size_allocatable_p(size_t size); diff --git a/ractor.c b/ractor.c index 9d34dd1b7d269a..f1366d93ba2275 100644 --- a/ractor.c +++ b/ractor.c @@ -1936,7 +1936,7 @@ vm_insert_ractor0(rb_vm_t *vm, rb_ractor_t *r, bool single_ractor_mode) VM_ASSERT(r == ruby_single_main_ractor); } else { - r->newobj_cache = rb_gc_ractor_cache_alloc(); + r->newobj_cache = rb_gc_ractor_cache_alloc(r); } } @@ -2036,7 +2036,7 @@ rb_ractor_main_alloc(void) r->loc = Qnil; r->name = Qnil; r->pub.self = Qnil; - r->newobj_cache = rb_gc_ractor_cache_alloc(); + r->newobj_cache = rb_gc_ractor_cache_alloc(r); ruby_single_main_ractor = r; return r; diff --git a/variable.c b/variable.c index 0dfc5cb7b83dfc..061084ec5bd3e6 100644 --- a/variable.c +++ b/variable.c @@ -1056,6 +1056,12 @@ generic_ivtbl_no_ractor_check(VALUE obj) return generic_ivtbl(obj, 0, false); } +struct st_table * +generic_ivtbl_get(void) +{ + return generic_iv_tbl_; +} + int rb_gen_ivtbl_get(VALUE obj, ID id, struct gen_ivtbl **ivtbl) {