diff --git a/.github/scripts/ci-build.sh b/.github/scripts/ci-build.sh index de3b70cc..8e23f7c9 100755 --- a/.github/scripts/ci-build.sh +++ b/.github/scripts/ci-build.sh @@ -12,6 +12,8 @@ fi build_type=$1 # plan to use plan=$2 +# moving vs non-moving +is_moving=$3 # helloworld.jl HELLO_WORLD_JL=$BINDING_PATH/.github/scripts/hello_world.jl @@ -23,9 +25,10 @@ if [ "$build_type" == "release" ]; then fi plan_feature=${plan,,} +moving_feature=${is_moving,,} cd $MMTK_JULIA_DIR/mmtk -cargo build --features $plan_feature $build_args +cargo build --features $plan_feature,$moving_feature $build_args cd $JULIA_PATH diff --git a/.github/scripts/ci-test-patching.sh b/.github/scripts/ci-test-patching.sh index 4090d779..c52042d1 100755 --- a/.github/scripts/ci-test-patching.sh +++ b/.github/scripts/ci-test-patching.sh @@ -24,6 +24,10 @@ declare -a tests_to_skip=( # This test checks GC logging '@test occursin("GC: pause", read(tmppath, String))' "$JULIA_PATH/test/misc.jl" + # These tests check for the number of stock GC threads (which we set to 0 with mmtk) + '@test (cpu_threads == 1 ? "1" : string(div(cpu_threads, 2))) ==' "$JULIA_PATH/test/cmdlineargs.jl" + '@test read(`$exename --gcthreads=2 -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl" + '@test read(`$exename -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl" # This seems to be a regression from upstream when we merge with upstream 43bf2c8. # The required string int.jl does not appear in the output even if I test with the stock Julia code. # I do not know what is wrong, but at this point, I dont want to spend time on it. diff --git a/.github/workflows/binding-tests.yml b/.github/workflows/binding-tests.yml index f5be174b..4507d96d 100644 --- a/.github/workflows/binding-tests.yml +++ b/.github/workflows/binding-tests.yml @@ -6,6 +6,9 @@ on: gc_plan: required: true type: string + moving: + required: true + type: string jobs: build-debug: @@ -18,7 +21,7 @@ jobs: ./.github/scripts/ci-setup.sh - name: Build Julia (Debug) run: | - ./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }} + ./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }} ${{ inputs.moving }} - name: Style check run: | ./.github/scripts/ci-style.sh @@ -37,7 +40,7 @@ jobs: ./.github/scripts/ci-test-patching.sh - name: Build Julia (Release) run: | - ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} + ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} ${{ inputs.moving }} - name: Test Julia run: | ./.github/scripts/ci-test-other.sh @@ -53,7 +56,7 @@ jobs: ./.github/scripts/ci-setup.sh - name: Build Julia (Release) run: | - ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} + ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} ${{ inputs.moving }} - name: Test Julia run: | ./.github/scripts/ci-test-stdlib.sh @@ -69,7 +72,7 @@ jobs: ./.github/scripts/ci-setup.sh - name: Build Julia (Release) run: | - ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} + ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} ${{ inputs.moving }} - name: Test Julia run: | ./.github/scripts/ci-test-LinearAlgebra.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30bf8e28..8341d3e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: fail-fast: false matrix: gc_plan: [Immix, StickyImmix] + moving: [Default, Non_Moving] uses: ./.github/workflows/binding-tests.yml with: gc_plan: ${{ matrix.gc_plan }} + moving: ${{ matrix.moving }} diff --git a/julia/mmtk_julia.c b/julia/mmtk_julia.c index 0e73f30a..187136a8 100644 --- a/julia/mmtk_julia.c +++ b/julia/mmtk_julia.c @@ -21,6 +21,8 @@ extern void mmtk_store_obj_size_c(void* obj, size_t size); extern void jl_gc_free_array(jl_array_t *a); extern size_t mmtk_get_obj_size(void* obj); extern void jl_rng_split(uint64_t to[JL_RNG_SIZE], uint64_t from[JL_RNG_SIZE]); +extern void _jl_free_stack(jl_ptls_t ptls, void *stkbuf, size_t bufsz); +extern void free_stack(void *stkbuf, size_t bufsz); extern jl_mutex_t finalizers_lock; extern void jl_gc_wait_for_the_world(jl_ptls_t* gc_all_tls_states, int gc_n_threads); extern void mmtk_block_thread_for_gc(void); @@ -114,6 +116,9 @@ static void mmtk_sweep_malloced_arrays(void) JL_NOTSAFEPOINT continue; } if (mmtk_is_live_object(ma->a)) { + // if the array has been forwarded, the reference needs to be updated + jl_array_t *maybe_forwarded = (jl_array_t*)mmtk_get_possibly_forwared(ma->a); + ma->a = maybe_forwarded; pma = &ma->next; } else { @@ -274,6 +279,19 @@ static void add_node_to_roots_buffer(RootsWorkClosure* closure, RootsWorkBuffer* } } +static void add_node_to_tpinned_roots_buffer(RootsWorkClosure* closure, RootsWorkBuffer* buf, size_t* buf_len, void* root) { + if (root == NULL) + return; + + buf->ptr[*buf_len] = root; + *buf_len += 1; + if (*buf_len >= buf->cap) { + RootsWorkBuffer new_buf = (closure->report_tpinned_nodes_func)(buf->ptr, *buf_len, buf->cap, closure->data, true); + *buf = new_buf; + *buf_len = 0; + } +} + void scan_vm_specific_roots(RootsWorkClosure* closure) { // Create a new buf @@ -302,10 +320,15 @@ void scan_vm_specific_roots(RootsWorkClosure* closure) // constants add_node_to_roots_buffer(closure, &buf, &len, jl_emptytuple_type); add_node_to_roots_buffer(closure, &buf, &len, cmpswap_names); - add_node_to_roots_buffer(closure, &buf, &len, jl_global_roots_table); + + // jl_global_roots_table must be transitively pinned + RootsWorkBuffer tpinned_buf = (closure->report_tpinned_nodes_func)((void**)0, 0, 0, closure->data, true); + size_t tpinned_len = 0; + add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, jl_global_roots_table); // Push the result of the work. (closure->report_nodes_func)(buf.ptr, len, buf.cap, closure->data, false); + (closure->report_tpinned_nodes_func)(tpinned_buf.ptr, tpinned_len, tpinned_buf.cap, closure->data, false); } JL_DLLEXPORT void scan_julia_exc_obj(void* obj_raw, void* closure, ProcessEdgeFn process_edge) { @@ -349,6 +372,107 @@ JL_DLLEXPORT void scan_julia_exc_obj(void* obj_raw, void* closure, ProcessEdgeFn } } +// number of stacks to always keep available per pool - from gc-stacks.c +#define MIN_STACK_MAPPINGS_PER_POOL 5 + +// if data is inlined inside the array object --- to->data needs to be updated when copying the array +void update_inlined_array(void* from, void* to) { + jl_value_t* jl_from = (jl_value_t*) from; + jl_value_t* jl_to = (jl_value_t*) to; + + uintptr_t tag_to = (uintptr_t)jl_typeof(jl_to); + jl_datatype_t *vt = (jl_datatype_t*)tag_to; + + if(vt->name == jl_array_typename) { + jl_array_t *a = (jl_array_t*)jl_from; + jl_array_t *b = (jl_array_t*)jl_to; + if (a->flags.how == 0 && mmtk_object_is_managed_by_mmtk(a->data)) { // a is inlined (a->data is an mmtk object) + size_t offset_of_data = ((size_t)a->data - a->offset*a->elsize) - (size_t)a; + if (offset_of_data > 0 && offset_of_data <= ARRAY_INLINE_NBYTES) { + b->data = (void*)((size_t) b + offset_of_data); + } + } + } +} + +// modified sweep_stack_pools from gc-stacks.c +void mmtk_sweep_stack_pools(void) +{ + // Stack sweeping algorithm: + // // deallocate stacks if we have too many sitting around unused + // for (stk in halfof(free_stacks)) + // free_stack(stk, pool_sz); + // // then sweep the task stacks + // for (t in live_tasks) + // if (!gc-marked(t)) + // stkbuf = t->stkbuf + // bufsz = t->bufsz + // if (stkbuf) + // push(free_stacks[sz], stkbuf) + for (int i = 0; i < jl_n_threads; i++) { + jl_ptls_t ptls2 = jl_all_tls_states[i]; + + // free half of stacks that remain unused since last sweep + for (int p = 0; p < JL_N_STACK_POOLS; p++) { + arraylist_t *al = &ptls2->heap.free_stacks[p]; + size_t n_to_free; + if (al->len > MIN_STACK_MAPPINGS_PER_POOL) { + n_to_free = al->len / 2; + if (n_to_free > (al->len - MIN_STACK_MAPPINGS_PER_POOL)) + n_to_free = al->len - MIN_STACK_MAPPINGS_PER_POOL; + } + else { + n_to_free = 0; + } + for (int n = 0; n < n_to_free; n++) { + void *stk = arraylist_pop(al); + free_stack(stk, pool_sizes[p]); + } + } + + arraylist_t *live_tasks = &ptls2->heap.live_tasks; + size_t n = 0; + size_t ndel = 0; + size_t l = live_tasks->len; + void **lst = live_tasks->items; + if (l == 0) + continue; + while (1) { + jl_task_t *t = (jl_task_t*)lst[n]; + if (mmtk_is_live_object(t)) { + jl_task_t *maybe_forwarded = (jl_task_t*)mmtk_get_possibly_forwared(t); + live_tasks->items[n] = maybe_forwarded; + t = maybe_forwarded; + assert(jl_is_task(t)); + if (t->stkbuf == NULL) + ndel++; // jl_release_task_stack called + else + n++; + } else { + ndel++; + void *stkbuf = t->stkbuf; + size_t bufsz = t->bufsz; + if (stkbuf) { + t->stkbuf = NULL; + _jl_free_stack(ptls2, stkbuf, bufsz); + } +#ifdef _COMPILER_TSAN_ENABLED_ + if (t->ctx.tsan_state) { + __tsan_destroy_fiber(t->ctx.tsan_state); + t->ctx.tsan_state = NULL; + } +#endif + } + if (n >= l - ndel) + break; + void *tmp = lst[n]; + lst[n] = lst[n + ndel]; + lst[n + ndel] = tmp; + } + live_tasks->len -= ndel; + } +} + #define jl_array_data_owner_addr(a) (((jl_value_t**)((char*)a + jl_array_data_owner_offset(jl_array_ndims(a))))) JL_DLLEXPORT void* get_stackbase(int16_t tid) { @@ -431,6 +555,7 @@ Julia_Upcalls mmtk_upcalls = (Julia_Upcalls) { .mmtk_jl_run_finalizers = mmtk_jl_run_finalizers, .jl_throw_out_of_memory_error = jl_throw_out_of_memory_error, .sweep_malloced_array = mmtk_sweep_malloced_arrays, + .sweep_stack_pools = mmtk_sweep_stack_pools, .wait_in_a_safepoint = mmtk_wait_in_a_safepoint, .exit_from_safepoint = mmtk_exit_from_safepoint, .jl_hrtime = jl_hrtime, @@ -442,5 +567,6 @@ Julia_Upcalls mmtk_upcalls = (Julia_Upcalls) { .arraylist_grow = (void (*)(void*, long unsigned int))arraylist_grow, .get_jl_gc_have_pending_finalizers = get_jl_gc_have_pending_finalizers, .scan_vm_specific_roots = scan_vm_specific_roots, + .update_inlined_array = update_inlined_array, .prepare_to_collect = jl_gc_prepare_to_collect, }; diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index be0729cf..5798a954 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -86,9 +86,9 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytemuck" -version = "1.14.0" +version = "1.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" +checksum = "a2ef034f05691a48569bd920a96c81b9d91bbad1ab5ac7c4616c1f6ef36cb79f" dependencies = [ "bytemuck_derive", ] @@ -122,9 +122,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.32" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41daef31d7a747c5c847246f36de49ced6f7403b4cdabc807a97b5cc184cda7a" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", @@ -215,9 +215,9 @@ checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "enum-map" @@ -252,16 +252,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "errno" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" -dependencies = [ - "libc", - "windows-sys", -] - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -273,9 +263,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd" +checksum = "1b3ba52851e73b46a4c3df1d89343741112003f0f6f13beb0dfac9e457c3fdcd" dependencies = [ "bitflags", "libc", @@ -292,9 +282,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" +checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" [[package]] name = "humantime" @@ -304,9 +294,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "iana-time-zone" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -337,38 +327,38 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ "hermit-abi", - "rustix", + "libc", "windows-sys", ] [[package]] name = "itertools" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" dependencies = [ "wasm-bindgen", ] @@ -381,15 +371,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.152" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libgit2-sys" -version = "0.16.1+1.7.1" +version = "0.16.2+1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2a2bb3680b094add03bb3732ec520ece34da31a8cd2d633d1389d0f0fb60d0c" +checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" dependencies = [ "cc", "libc", @@ -399,9 +389,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.14" +version = "1.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050" +checksum = "037731f5d3aaa87a5675e895b63ddff1a87624bc29f77004ea829809654e48f6" dependencies = [ "cc", "libc", @@ -409,12 +399,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "linux-raw-sys" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" - [[package]] name = "lock_api" version = "0.4.11" @@ -493,6 +477,7 @@ dependencies = [ "lazy_static", "libc", "log", + "memoffset", "mmtk", "thread-id", ] @@ -519,9 +504,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", ] @@ -550,9 +535,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.6" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f200d8d83c44a45b21764d1916299752ca035d15ecd46faca3e9a2a2bf6ad06" +checksum = "219c0dcc30b6a27553f9cc242972b67f75b60eb0db71f0b5462f38b058c41546" dependencies = [ "memchr", "thiserror", @@ -653,9 +638,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -677,19 +662,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "0.38.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] - [[package]] name = "rustversion" version = "1.0.14" @@ -802,18 +774,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", @@ -897,9 +869,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -907,9 +879,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" dependencies = [ "bumpalo", "log", @@ -922,9 +894,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -932,9 +904,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", @@ -945,9 +917,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" [[package]] name = "winapi" diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index fe5fac16..ea0fb526 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -10,7 +10,7 @@ edition = "2018" [package.metadata.julia] # Our CI matches the following line and extract mmtk/julia. If this line is updated, please check ci yaml files and make sure it works. julia_repo = "https://github.com/mmtk/julia.git" -julia_version = "22524a81f9920a8ecc70f195d53b145761de2fa0" +julia_version = "861f151c7fbec1d8431a804795bd3d3598b77e3b" [lib] crate-type = ["cdylib"] @@ -39,18 +39,20 @@ enum-map = ">=2.1" atomic = "0.4.6" chrono = "*" thread-id = "*" +memoffset = "*" # ykstackmaps = { git = "https://github.com/udesou/ykstackmaps.git", branch = "udesou-master", version = "*" } [features] -default = ["mmtk/vm_space", "julia_copy_stack"] +default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning"] # Plans nogc = [] -immix = ["non_moving_immix"] -stickyimmix = ["non_moving_immix"] +immix = [] +stickyimmix = ["mmtk/sticky_immix_non_moving_nursery", "mmtk/immix_smaller_block"] marksweep = [] +object_pinning = ["mmtk/object_pinning"] -# TODO remove this when we properly support moving -non_moving_immix = ["mmtk/immix_non_moving", "mmtk/immix_smaller_block"] +# This feature disables moving +non_moving = ["mmtk/immix_non_moving", "mmtk/immix_smaller_block"] julia_copy_stack = [] diff --git a/mmtk/api/mmtk.h b/mmtk/api/mmtk.h index 9ddd411b..3a7f7de6 100644 --- a/mmtk/api/mmtk.h +++ b/mmtk/api/mmtk.h @@ -22,6 +22,7 @@ typedef struct { typedef struct { RootsWorkBuffer (*report_edges_func)(void** buf, size_t size, size_t cap, void* data, bool renew); RootsWorkBuffer (*report_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew); + RootsWorkBuffer (*report_tpinned_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew); void* data; } RootsWorkClosure; @@ -47,6 +48,8 @@ extern bool mmtk_is_mapped_address(void* addr); extern int mmtk_object_is_managed_by_mmtk(void* addr); extern void mmtk_runtime_panic(void); extern void mmtk_unreachable(void); +extern unsigned char mmtk_pin_object(void* obj); +extern bool mmtk_is_pinned(void* obj); extern void mmtk_set_vm_space(void* addr, size_t size); extern void mmtk_immortal_region_post_alloc(void* addr, size_t size); @@ -72,6 +75,7 @@ typedef struct { void (* mmtk_jl_run_finalizers) (void* tls); void (* jl_throw_out_of_memory_error) (void); void (* sweep_malloced_array) (void); + void (* sweep_stack_pools) (void); void (* wait_in_a_safepoint) (void); void (* exit_from_safepoint) (int8_t old_state); uint64_t (* jl_hrtime) (void); @@ -83,6 +87,7 @@ typedef struct { void (*arraylist_grow)(void* a, size_t n); int* (*get_jl_gc_have_pending_finalizers)(void); void (*scan_vm_specific_roots)(RootsWorkClosure* closure); + void (*update_inlined_array) (void* from, void* to); void (*prepare_to_collect)(void); } Julia_Upcalls; @@ -103,6 +108,7 @@ extern void mmtk_run_finalizers_for_obj(void* obj); extern void mmtk_run_finalizers(bool at_exit); extern void mmtk_gc_poll(void *tls); extern void mmtk_julia_copy_stack_check(int copy_stack); +extern void* mmtk_get_possibly_forwared(void* object); /** * VM Accounting diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index a8eaa11a..d74b3fa5 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -483,3 +483,55 @@ pub extern "C" fn mmtk_get_obj_size(obj: ObjectReference) -> usize { addr_size.load::() as usize } } + +#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))] +#[no_mangle] +pub extern "C" fn mmtk_pin_object(object: ObjectReference) -> bool { + if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { + memory_manager::pin_object::(object) + } else { + warn!("Object is not managed by mmtk - (un)pinning it via this function isn't supported."); + false + } +} + +#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))] +#[no_mangle] +pub extern "C" fn mmtk_unpin_object(object: ObjectReference) -> bool { + if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { + memory_manager::unpin_object::(object) + } else { + warn!("Object is not managed by mmtk - (un)pinning it via this function isn't supported."); + false + } +} + +#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))] +#[no_mangle] +pub extern "C" fn mmtk_is_pinned(object: ObjectReference) -> bool { + if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { + memory_manager::is_pinned::(object) + } else { + warn!("Object is not managed by mmtk - checking via this function isn't supported."); + false + } +} + +// If the `non-moving` feature is selected, pinning/unpinning is a noop and simply returns false +#[cfg(all(feature = "object_pinning", feature = "non_moving"))] +#[no_mangle] +pub extern "C" fn mmtk_pin_object(_object: ObjectReference) -> bool { + false +} + +#[cfg(all(feature = "object_pinning", feature = "non_moving"))] +#[no_mangle] +pub extern "C" fn mmtk_unpin_object(_object: ObjectReference) -> bool { + false +} + +#[cfg(all(feature = "object_pinning", feature = "non_moving"))] +#[no_mangle] +pub extern "C" fn mmtk_is_pinned(_object: ObjectReference) -> bool { + false +} diff --git a/mmtk/src/edges.rs b/mmtk/src/edges.rs index 04f751c1..6f7d42af 100644 --- a/mmtk/src/edges.rs +++ b/mmtk/src/edges.rs @@ -215,6 +215,13 @@ pub struct RootsWorkClosure { factory_ptr: *mut libc::c_void, renew: bool, ) -> RootsWorkBuffer, + pub report_tpinned_nodes_func: extern "C" fn( + buf: *mut ObjectReference, + size: usize, + cap: usize, + factory_ptr: *mut libc::c_void, + renew: bool, + ) -> RootsWorkBuffer, pub factory_ptr: *mut libc::c_void, } @@ -262,10 +269,31 @@ impl RootsWorkClosure { } } + extern "C" fn report_tpinned_nodes>( + buf: *mut ObjectReference, + size: usize, + cap: usize, + factory_ptr: *mut libc::c_void, + renew: bool, + ) -> RootsWorkBuffer { + if !buf.is_null() { + let buf = unsafe { Vec::::from_raw_parts(buf, size, cap) }; + let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; + factory.create_process_tpinning_roots_work(buf); + } + + if renew { + RootsWorkBuffer::new() + } else { + RootsWorkBuffer::empty() + } + } + pub fn from_roots_work_factory>(factory: &mut F) -> Self { RootsWorkClosure { report_edges_func: Self::report_simple_edges::, report_nodes_func: Self::report_nodes::, + report_tpinned_nodes_func: Self::report_tpinned_nodes::, factory_ptr: factory as *mut F as *mut libc::c_void, } } diff --git a/mmtk/src/julia_finalizer.rs b/mmtk/src/julia_finalizer.rs index 754fe81d..fb8534ab 100644 --- a/mmtk/src/julia_finalizer.rs +++ b/mmtk/src/julia_finalizer.rs @@ -186,6 +186,9 @@ fn mark_finlist(list: &mut ArrayListT, start: usize, tracer: &m let mut i = start; while i < list.len { let cur = list.get(i); + let cur_i = i; + let mut cur_tag: usize = 0; + if cur.is_zero() { i += 1; continue; @@ -195,6 +198,7 @@ fn mark_finlist(list: &mut ArrayListT, start: usize, tracer: &m // Skip next i += 1; debug_assert!(i < list.len); + cur_tag = 1; gc_ptr_clear_tag(cur, 1) } else { ObjectReference::from_raw_address(cur) @@ -205,10 +209,11 @@ fn mark_finlist(list: &mut ArrayListT, start: usize, tracer: &m } let traced = tracer.trace_object(new_obj); - debug_assert_eq!( - traced, new_obj, - "Object is moved -- we need to save the new object back to the finalizer list" - ); + // if object has moved, update the list applying the tag + list.set(cur_i, unsafe { + Address::from_usize(traced.to_raw_address() | cur_tag) + }); + i += 1; } } diff --git a/mmtk/src/julia_scanning.rs b/mmtk/src/julia_scanning.rs index 3ae809da..28d31f61 100644 --- a/mmtk/src/julia_scanning.rs +++ b/mmtk/src/julia_scanning.rs @@ -1,3 +1,5 @@ +use crate::api::mmtk_is_pinned; +use crate::api::mmtk_object_is_managed_by_mmtk; use crate::edges::JuliaVMEdge; use crate::edges::OffsetEdge; use crate::julia_types::*; @@ -5,6 +7,7 @@ use crate::object_model::mmtk_jl_array_ndims; use crate::JuliaVM; use crate::JULIA_BUFF_TAG; use crate::UPCALLS; +use memoffset::offset_of; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::edge_shape::SimpleEdge; use mmtk::vm::EdgeVisitor; @@ -12,6 +15,8 @@ use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; const JL_MAX_TAGS: usize = 64; // from vm/julia/src/jl_exports.h +const OFFSET_OF_INLINED_SPACE_IN_MODULE: usize = + offset_of!(mmtk_jl_module_t, usings) + offset_of!(mmtk_arraylist_t, _space); extern "C" { pub static jl_simplevector_type: *const mmtk_jl_datatype_t; @@ -86,7 +91,13 @@ pub unsafe fn scan_julia_object>(obj: Address, clos let array = obj.to_ptr::(); let flags = (*array).flags; - if flags.how_custom() == 1 { + if flags.how_custom() == 0 { + // data is inlined, or a foreign pointer we don't manage + // if data is inlined (i.e. it is an internal pointer) and the array moves, + // a->data is currently updated when copying the array since there may be other hidden + // fields before the inlined data affecting the offset in which a->data points to + // see jl_array_t in julia.h + } else if flags.how_custom() == 1 { // julia-allocated buffer that needs to be marked let offset = (*array).offset as usize * (*array).elsize as usize; let data_addr = ::std::ptr::addr_of!((*array).data); @@ -97,6 +108,19 @@ pub unsafe fn scan_julia_object>(obj: Address, clos } else if flags.how_custom() == 3 { // has a pointer to the object that owns the data let owner_addr = mmtk_jl_array_data_owner_addr(array); + + // to avoid having to update a->data, which requires introspecting the owner object + // we simply expect that both owner and buffers are pinned when in a moving GC + #[cfg(not(feature = "non_moving"))] + debug_assert!( + (mmtk_object_is_managed_by_mmtk(owner_addr.load()) + && mmtk_is_pinned(owner_addr.load()) + || !(mmtk_object_is_managed_by_mmtk(owner_addr.load()))), + "Owner ({:?}) may move (is_pinned = {}), a->data may become outdated!", + owner_addr.load::(), + mmtk_is_pinned(owner_addr.load()) + ); + process_edge(closure, owner_addr); return; } @@ -195,6 +219,16 @@ pub unsafe fn scan_julia_object>(obj: Address, clos } process_edge(closure, Address::from_ptr(bindings_edge)); + // m.usings.items may be inlined in the module when the array list size <= AL_N_INLINE (cf. arraylist_new) + // In that case it may be an mmtk object and not a malloced address. + // If it is an mmtk object, (*m).usings.items will then be an internal pointer to the module + // which means we will need to trace and update it if the module moves + if mmtk_object_is_managed_by_mmtk((*m).usings.items as usize) { + let offset = OFFSET_OF_INLINED_SPACE_IN_MODULE; + let slot = Address::from_ptr(::std::ptr::addr_of!((*m).usings.items)); + process_offset_edge(closure, slot, offset); + } + let nusings = (*m).usings.len; if nusings != 0 { let mut objary_begin = Address::from_mut_ptr((*m).usings.items); @@ -394,6 +428,16 @@ pub fn process_edge>(closure: &mut EV, slot: Addres simple_edge.load(), simple_edge ); + + // captures wrong edges before creating the work + debug_assert!( + simple_edge.load().to_raw_address().as_usize() % 16 == 0 + || simple_edge.load().to_raw_address().as_usize() % 8 == 0, + "Object {:?} in slot {:?} is not aligned to 8 or 16", + simple_edge.load(), + simple_edge + ); + closure.visit_edge(JuliaVMEdge::Simple(simple_edge)); } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 86ad3b31..d0720c29 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -102,6 +102,7 @@ pub struct Julia_Upcalls { pub mmtk_jl_run_finalizers: extern "C" fn(tls: OpaquePointer), pub jl_throw_out_of_memory_error: extern "C" fn(), pub mmtk_sweep_malloced_array: extern "C" fn(), + pub mmtk_sweep_stack_pools: extern "C" fn(), pub wait_in_a_safepoint: extern "C" fn(), pub exit_from_safepoint: extern "C" fn(old_state: i8), pub jl_hrtime: extern "C" fn() -> u64, @@ -113,6 +114,7 @@ pub struct Julia_Upcalls { pub arraylist_grow: extern "C" fn(Address, usize), pub get_jl_gc_have_pending_finalizers: extern "C" fn() -> *mut i32, pub scan_vm_specific_roots: extern "C" fn(closure: *mut crate::edges::RootsWorkClosure), + pub update_inlined_array: extern "C" fn(to: Address, from: Address), pub prepare_to_collect: extern "C" fn(), } diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index a7723839..ed838ff1 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -4,8 +4,9 @@ use crate::julia_scanning::{ jl_task_type, mmtk_jl_array_len, mmtk_jl_array_ndimwords, mmtk_jl_tparam0, mmtk_jl_typeof, mmtk_jl_typetagof, }; -use crate::julia_types::*; +use crate::{julia_types::*, UPCALLS}; use crate::{JuliaVM, JULIA_BUFF_TAG, JULIA_HEADER_SIZE}; +use log::trace; use mmtk::util::copy::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::ObjectModel; @@ -20,6 +21,10 @@ pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBit pub(crate) const MARKING_METADATA_SPEC: VMLocalMarkBitSpec = VMLocalMarkBitSpec::side_after(LOS_METADATA_SPEC.as_spec()); +#[cfg(feature = "object_pinning")] +pub(crate) const LOCAL_PINNING_METADATA_BITS_SPEC: VMLocalPinningBitSpec = + VMLocalPinningBitSpec::side_after(MARKING_METADATA_SPEC.as_spec()); + // pub(crate) const LOCAL_FORWARDING_POINTER_METADATA_SPEC: VMLocalForwardingPointerSpec = // VMLocalForwardingPointerSpec::side_after(MARKING_METADATA_SPEC.as_spec()); @@ -34,20 +39,78 @@ pub(crate) const LOS_METADATA_SPEC: VMLocalLOSMarkNurserySpec = impl ObjectModel for VMObjectModel { const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = LOGGING_SIDE_METADATA_SPEC; const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = - VMLocalForwardingPointerSpec::in_header(0); + VMLocalForwardingPointerSpec::in_header(-64); + + #[cfg(feature = "object_pinning")] + const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = + VMLocalForwardingBitsSpec::side_after(LOCAL_PINNING_METADATA_BITS_SPEC.as_spec()); + #[cfg(not(feature = "object_pinning"))] const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = - VMLocalForwardingBitsSpec::in_header(0); + VMLocalForwardingBitsSpec::side_after(MARKING_METADATA_SPEC.as_spec()); + const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = MARKING_METADATA_SPEC; const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = LOS_METADATA_SPEC; const UNIFIED_OBJECT_REFERENCE_ADDRESS: bool = false; const OBJECT_REF_OFFSET_LOWER_BOUND: isize = 0; + #[cfg(feature = "object_pinning")] + const LOCAL_PINNING_BIT_SPEC: VMLocalPinningBitSpec = LOCAL_PINNING_METADATA_BITS_SPEC; + fn copy( - _from: ObjectReference, - _semantics: CopySemantics, - _copy_context: &mut GCWorkerCopyContext, + from: ObjectReference, + semantics: CopySemantics, + copy_context: &mut GCWorkerCopyContext, ) -> ObjectReference { - unimplemented!() + let bytes = Self::get_current_size(from); + let from_start_ref = ObjectReference::from_raw_address(Self::ref_to_object_start(from)); + let header_offset = + from.to_raw_address().as_usize() - from_start_ref.to_raw_address().as_usize(); + + let dst = if header_offset == 8 { + // regular object + copy_context.alloc_copy(from_start_ref, bytes, 16, 8, semantics) + } else if header_offset == 16 { + // buffer should not be copied + unimplemented!(); + } else { + unimplemented!() + }; + + let src = Self::ref_to_object_start(from); + unsafe { + std::ptr::copy_nonoverlapping::(src.to_ptr(), dst.to_mut_ptr(), bytes); + } + let to_obj = ObjectReference::from_raw_address(dst + header_offset); + + trace!("Copying object from {} to {}", from, to_obj); + + copy_context.post_copy(to_obj, bytes, semantics); + + unsafe { + let vt = mmtk_jl_typeof(from.to_raw_address()); + + if (*vt).name == jl_array_typename { + ((*UPCALLS).update_inlined_array)(from.to_raw_address(), to_obj.to_raw_address()) + } + } + + // zero from_obj (for debugging purposes) + #[cfg(debug_assertions)] + { + use atomic::Ordering; + unsafe { + libc::memset(from_start_ref.to_raw_address().to_mut_ptr(), 0, bytes); + } + + Self::LOCAL_FORWARDING_BITS_SPEC.store_atomic::( + from, + 0b10 as u8, // BEING_FORWARDED + None, + Ordering::SeqCst, + ); + } + + to_obj } fn copy_to(_from: ObjectReference, _to: ObjectReference, _region: Address) -> Address { @@ -101,9 +164,8 @@ impl ObjectModel for VMObjectModel { } #[inline(always)] - fn ref_to_header(_object: ObjectReference) -> Address { - unreachable!() - // object.to_raw_address() - 8 + fn ref_to_header(object: ObjectReference) -> Address { + object.to_raw_address() } fn dump_object(_object: ObjectReference) { diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 2976764d..e3a000a3 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -23,12 +23,24 @@ impl Scanning for VMScanning { mut factory: impl RootsWorkFactory, ) { // This allows us to reuse mmtk_scan_gcstack which expectes an EdgeVisitor + // Push the nodes as they need to be transitively pinned struct EdgeBuffer { - pub buffer: Vec, + pub buffer: Vec, } impl mmtk::vm::EdgeVisitor for EdgeBuffer { fn visit_edge(&mut self, edge: JuliaVMEdge) { - self.buffer.push(edge); + match edge { + JuliaVMEdge::Simple(se) => { + let slot = se.as_address(); + let object = unsafe { slot.load::() }; + if !object.is_null() { + self.buffer.push(object); + } + } + JuliaVMEdge::Offset(_) => { + unimplemented!() // transitively pinned roots in Julia only come from the stack + } + } } } @@ -37,22 +49,44 @@ impl Scanning for VMScanning { use mmtk::util::Address; let ptls: &mut mmtk__jl_tls_states_t = unsafe { std::mem::transmute(mutator.mutator_tls) }; - let mut edge_buffer = EdgeBuffer { buffer: vec![] }; + let mut edge_buffer = EdgeBuffer { buffer: vec![] }; // need to be tpinned as they're all from the shadow stack let mut node_buffer = vec![]; // Scan thread local from ptls: See gc_queue_thread_local in gc.c - let mut root_scan_task = |task: *const mmtk__jl_task_t| { + let mut root_scan_task = |task: *const mmtk__jl_task_t, task_is_root: bool| { if !task.is_null() { unsafe { crate::julia_scanning::mmtk_scan_gcstack(task, &mut edge_buffer); } - node_buffer.push(ObjectReference::from_raw_address(Address::from_ptr(task))); + if task_is_root { + // captures wrong root nodes before creating the work + debug_assert!( + Address::from_ptr(task).as_usize() % 16 == 0 + || Address::from_ptr(task).as_usize() % 8 == 0, + "root node {:?} is not aligned to 8 or 16", + Address::from_ptr(task) + ); + + node_buffer.push(ObjectReference::from_raw_address(Address::from_ptr(task))); + } } }; - root_scan_task(ptls.root_task); - root_scan_task(ptls.current_task as *mut mmtk__jl_task_t); - root_scan_task(ptls.next_task); - root_scan_task(ptls.previous_task); + root_scan_task(ptls.root_task, true); + + // need to iterate over live tasks as well to process their shadow stacks + // we should not set the task themselves as roots as we will know which ones are still alive after GC + let mut i = 0; + while i < ptls.heap.live_tasks.len { + let mut task_address = Address::from_ptr(ptls.heap.live_tasks.items); + task_address = task_address.shift::
(i as isize); + let task = unsafe { task_address.load::<*const mmtk_jl_task_t>() }; + root_scan_task(task, false); + i += 1; + } + + root_scan_task(ptls.current_task as *mut mmtk__jl_task_t, true); + root_scan_task(ptls.next_task, true); + root_scan_task(ptls.previous_task, true); if !ptls.previous_exception.is_null() { node_buffer.push(ObjectReference::from_raw_address(Address::from_mut_ptr( ptls.previous_exception, @@ -71,6 +105,15 @@ impl Scanning for VMScanning { let njlvals = mmtk_jl_bt_num_jlvals(bt_entry); for j in 0..njlvals { let bt_entry_value = mmtk_jl_bt_entry_jlvalue(bt_entry, j); + + // captures wrong root nodes before creating the work + debug_assert!( + bt_entry_value.to_raw_address().as_usize() % 16 == 0 + || bt_entry_value.to_raw_address().as_usize() % 8 == 0, + "root node {:?} is not aligned to 8 or 16", + bt_entry_value + ); + node_buffer.push(bt_entry_value); } i += bt_entry_size; @@ -80,12 +123,12 @@ impl Scanning for VMScanning { // Push work const CAPACITY_PER_PACKET: usize = 4096; - for edges in edge_buffer + for tpinning_roots in edge_buffer .buffer .chunks(CAPACITY_PER_PACKET) .map(|c| c.to_vec()) { - factory.create_process_edge_roots_work(edges); + factory.create_process_tpinning_roots_work(tpinning_roots); } for nodes in node_buffer.chunks(CAPACITY_PER_PACKET).map(|c| c.to_vec()) { factory.create_process_pinning_roots_work(nodes); @@ -111,11 +154,11 @@ impl Scanning for VMScanning { process_object(object, edge_visitor); } fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { - let sweep_malloced_arrays_work = SweepMallocedArrays::new(); + let sweep_vm_specific_work = SweepVMSpecific::new(); memory_manager::add_work_packet( &SINGLETON, WorkBucketStage::Compact, - sweep_malloced_arrays_work, + sweep_vm_specific_work, ); } fn supports_return_barrier() -> bool { @@ -150,20 +193,21 @@ pub fn process_object>(object: ObjectReference, clo } // Sweep malloced arrays work -pub struct SweepMallocedArrays { +pub struct SweepVMSpecific { swept: bool, } -impl SweepMallocedArrays { +impl SweepVMSpecific { pub fn new() -> Self { Self { swept: false } } } -impl GCWork for SweepMallocedArrays { +impl GCWork for SweepVMSpecific { fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { - // call sweep malloced arrays from UPCALLS + // call sweep malloced arrays and sweep stack pools from UPCALLS unsafe { ((*UPCALLS).mmtk_sweep_malloced_array)() } + unsafe { ((*UPCALLS).mmtk_sweep_stack_pools)() } self.swept = true; } } diff --git a/mmtk/src/util.rs b/mmtk/src/util.rs index 13bc4456..6d702ff8 100644 --- a/mmtk/src/util.rs +++ b/mmtk/src/util.rs @@ -1,4 +1,5 @@ use enum_map::Enum; +use mmtk::util::ObjectReference; #[repr(i32)] #[derive(Clone, Copy, Debug, Enum, PartialEq, Hash, Eq)] @@ -143,3 +144,11 @@ pub extern "C" fn mmtk_julia_copy_stack_check(c_flag_is_defined: bool) { panic!("COPY_STACK flag has not been defined in C, but `julia_copy_stack` feature has been set.") } } + +#[no_mangle] +pub extern "C" fn mmtk_get_possibly_forwared(object: ObjectReference) -> ObjectReference { + match object.get_forwarded_object() { + Some(forwarded) => forwarded, + None => object, + } +}