Supporting moving immix (#93)

This PR adds support for (partially) moving objects in Julia and should
be merged with mmtk/julia#27 and
mmtk/mmtk-core#897.
- It adds support for pinning/unpinning objects and checking if an
object is pinned (the implementation uses a pin bit).
(`mmtk_pin_object`, `mmtk_unpin_object` and `mmtk_is_pinned`)
- It adds support for providing transitively pinned (`tpinned`) roots .
- It implements the `copy` function in `object_model.rs`. Note that
arrays with inlined data must be treated specially, as their `a->data`
pointer needs to be updated after copying.
- It uses Julia's GC bits to store forwarding information and the
object's header to store the forwarding pointer.
- Currently, all stack roots are transitively pinned. Note that we also
need to traverse the `tls->live_tasks` to make sure that any stack root
from these tasks are transitively pinned.
- `scan_julia_object` had to be adapted to cover a few corner cases:
- when an array object contains a pointer to the owner of the data,
`a->data` needs to be updated in case the owner moves.
- the `using` field inside a `jl_module_t` may also be inlined inside
the module, and if that's the case, we need to make sure that field is
updated if the module moves.
- when marking finalizers, traversing the list of malloced arrays, and
the list of live tasks at the end of GC, we need to updated these lists
with objects that have possibly been moved.
- Added a few debug assertions to capture scanning of misaligned objects
and roots.

NB: I've only tested moving immix; sticky immix is still non-moving.

---------

Co-authored-by: Luis Eduardo de Souza Amorim <[email protected]>
Co-authored-by: Luis Eduardo de Souza Amorim <[email protected]>
Co-authored-by: mmtkgc-bot <[email protected]>

.github/scripts/ci-build.sh

@@ -12,6 +12,8 @@ fi
 build_type=$1
 # plan to use
 plan=$2
+# moving vs non-moving
+is_moving=$3
 
 # helloworld.jl
 HELLO_WORLD_JL=$BINDING_PATH/.github/scripts/hello_world.jl
@@ -23,9 +25,10 @@ if [ "$build_type" == "release" ]; then
 fi
 
 plan_feature=${plan,,}
+moving_feature=${is_moving,,}
 
 cd $MMTK_JULIA_DIR/mmtk
-cargo build --features $plan_feature $build_args
+cargo build --features $plan_feature,$moving_feature $build_args
 
 cd $JULIA_PATH
 

.github/scripts/ci-test-patching.sh

@@ -24,6 +24,10 @@ declare -a tests_to_skip=(
     # This test checks GC logging
     '@test occursin("GC: pause", read(tmppath, String))' "$JULIA_PATH/test/misc.jl"
 
+    # These tests check for the number of stock GC threads (which we set to 0 with mmtk)
+    '@test (cpu_threads == 1 ? "1" : string(div(cpu_threads, 2))) ==' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test read(`$exename --gcthreads=2 -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test read(`$exename -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
     # This seems to be a regression from upstream when we merge with upstream 43bf2c8.
     # The required string int.jl does not appear in the output even if I test with the stock Julia code.
     # I do not know what is wrong, but at this point, I dont want to spend time on it.

.github/workflows/binding-tests.yml

@@ -6,6 +6,9 @@ on:
       gc_plan:
         required: true
         type: string
+      moving:
+        required: true
+        type: string
 
 jobs:
   build-debug:
@@ -18,7 +21,7 @@ jobs:
           ./.github/scripts/ci-setup.sh
       - name: Build Julia (Debug)
         run: |
-          ./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }}
+          ./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }} ${{ inputs.moving }}
       - name: Style check
         run: |
           ./.github/scripts/ci-style.sh
@@ -37,7 +40,7 @@ jobs:
           ./.github/scripts/ci-test-patching.sh
       - name: Build Julia (Release)
         run: |
-          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }}
+          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} ${{ inputs.moving }}
       - name: Test Julia
         run: |
           ./.github/scripts/ci-test-other.sh
@@ -53,7 +56,7 @@ jobs:
           ./.github/scripts/ci-setup.sh
       - name: Build Julia (Release)
         run: |
-          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }}
+          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} ${{ inputs.moving }}
       - name: Test Julia
         run: |
           ./.github/scripts/ci-test-stdlib.sh
@@ -69,7 +72,7 @@ jobs:
           ./.github/scripts/ci-setup.sh
       - name: Build Julia (Release)
         run: |
-          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }}
+          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} ${{ inputs.moving }}
       - name: Test Julia
         run: |
           ./.github/scripts/ci-test-LinearAlgebra.sh

.github/workflows/ci.yml

@@ -18,6 +18,8 @@ jobs:
       fail-fast: false
       matrix:
         gc_plan: [Immix, StickyImmix]
+        moving: [Default, Non_Moving]
     uses: ./.github/workflows/binding-tests.yml
     with:
       gc_plan: ${{ matrix.gc_plan }}
+      moving: ${{ matrix.moving }}

julia/mmtk_julia.c

@@ -21,6 +21,8 @@ extern void mmtk_store_obj_size_c(void* obj, size_t size);
 extern void jl_gc_free_array(jl_array_t *a);
 extern size_t mmtk_get_obj_size(void* obj);
 extern void jl_rng_split(uint64_t to[JL_RNG_SIZE], uint64_t from[JL_RNG_SIZE]);
+extern void _jl_free_stack(jl_ptls_t ptls, void *stkbuf, size_t bufsz);
+extern void free_stack(void *stkbuf, size_t bufsz);
 extern jl_mutex_t finalizers_lock;
 extern void jl_gc_wait_for_the_world(jl_ptls_t* gc_all_tls_states, int gc_n_threads);
 extern void mmtk_block_thread_for_gc(void);
@@ -114,6 +116,9 @@ static void mmtk_sweep_malloced_arrays(void) JL_NOTSAFEPOINT
                 continue;
             }
             if (mmtk_is_live_object(ma->a)) {
+                // if the array has been forwarded, the reference needs to be updated
+                jl_array_t *maybe_forwarded = (jl_array_t*)mmtk_get_possibly_forwared(ma->a);
+                ma->a = maybe_forwarded;
                 pma = &ma->next;
             }
             else {
@@ -274,6 +279,19 @@ static void add_node_to_roots_buffer(RootsWorkClosure* closure, RootsWorkBuffer*
     }
 }
 
+static void add_node_to_tpinned_roots_buffer(RootsWorkClosure* closure, RootsWorkBuffer* buf, size_t* buf_len, void* root) {
+    if (root == NULL)
+        return;
+
+    buf->ptr[*buf_len] = root;
+    *buf_len += 1;
+    if (*buf_len >= buf->cap) {
+        RootsWorkBuffer new_buf = (closure->report_tpinned_nodes_func)(buf->ptr, *buf_len, buf->cap, closure->data, true);
+        *buf = new_buf;
+        *buf_len = 0;
+    }
+}
+
 void scan_vm_specific_roots(RootsWorkClosure* closure)
 {
     // Create a new buf
@@ -302,10 +320,15 @@ void scan_vm_specific_roots(RootsWorkClosure* closure)
     // constants
     add_node_to_roots_buffer(closure, &buf, &len, jl_emptytuple_type);
     add_node_to_roots_buffer(closure, &buf, &len, cmpswap_names);
-    add_node_to_roots_buffer(closure, &buf, &len, jl_global_roots_table);
+
+    // jl_global_roots_table must be transitively pinned 
+    RootsWorkBuffer tpinned_buf = (closure->report_tpinned_nodes_func)((void**)0, 0, 0, closure->data, true);
+    size_t tpinned_len = 0;
+    add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, jl_global_roots_table);
 
     // Push the result of the work.
     (closure->report_nodes_func)(buf.ptr, len, buf.cap, closure->data, false);
+    (closure->report_tpinned_nodes_func)(tpinned_buf.ptr, tpinned_len, tpinned_buf.cap, closure->data, false);
 }
 
 JL_DLLEXPORT void scan_julia_exc_obj(void* obj_raw, void* closure, ProcessEdgeFn process_edge) {
@@ -349,6 +372,107 @@ JL_DLLEXPORT void scan_julia_exc_obj(void* obj_raw, void* closure, ProcessEdgeFn
     }
 }
 
+// number of stacks to always keep available per pool - from gc-stacks.c
+#define MIN_STACK_MAPPINGS_PER_POOL 5
+
+// if data is inlined inside the array object --- to->data needs to be updated when copying the array
+void update_inlined_array(void* from, void* to) {
+    jl_value_t* jl_from = (jl_value_t*) from;
+    jl_value_t* jl_to = (jl_value_t*) to;
+
+    uintptr_t tag_to = (uintptr_t)jl_typeof(jl_to);
+    jl_datatype_t *vt = (jl_datatype_t*)tag_to;
+
+    if(vt->name == jl_array_typename) {
+        jl_array_t *a = (jl_array_t*)jl_from;
+        jl_array_t *b = (jl_array_t*)jl_to;
+        if (a->flags.how == 0 && mmtk_object_is_managed_by_mmtk(a->data)) { // a is inlined (a->data is an mmtk object)
+            size_t offset_of_data = ((size_t)a->data - a->offset*a->elsize) - (size_t)a;
+            if (offset_of_data > 0 && offset_of_data <= ARRAY_INLINE_NBYTES) {
+                b->data = (void*)((size_t) b + offset_of_data);
+            }
+        }
+    }
+}
+
+// modified sweep_stack_pools from gc-stacks.c
+void mmtk_sweep_stack_pools(void)
+{
+    // Stack sweeping algorithm:
+    //    // deallocate stacks if we have too many sitting around unused
+    //    for (stk in halfof(free_stacks))
+    //        free_stack(stk, pool_sz);
+    //    // then sweep the task stacks
+    //    for (t in live_tasks)
+    //        if (!gc-marked(t))
+    //            stkbuf = t->stkbuf
+    //            bufsz = t->bufsz
+    //            if (stkbuf)
+    //                push(free_stacks[sz], stkbuf)
+    for (int i = 0; i < jl_n_threads; i++) {
+        jl_ptls_t ptls2 = jl_all_tls_states[i];
+
+        // free half of stacks that remain unused since last sweep
+        for (int p = 0; p < JL_N_STACK_POOLS; p++) {
+            arraylist_t *al = &ptls2->heap.free_stacks[p];
+            size_t n_to_free;
+            if (al->len > MIN_STACK_MAPPINGS_PER_POOL) {
+                n_to_free = al->len / 2;
+                if (n_to_free > (al->len - MIN_STACK_MAPPINGS_PER_POOL))
+                    n_to_free = al->len - MIN_STACK_MAPPINGS_PER_POOL;
+            }
+            else {
+                n_to_free = 0;
+            }
+            for (int n = 0; n < n_to_free; n++) {
+                void *stk = arraylist_pop(al);
+                free_stack(stk, pool_sizes[p]);
+            }
+        }
+
+        arraylist_t *live_tasks = &ptls2->heap.live_tasks;
+        size_t n = 0;
+        size_t ndel = 0;
+        size_t l = live_tasks->len;
+        void **lst = live_tasks->items;
+        if (l == 0)
+            continue;
+        while (1) {
+            jl_task_t *t = (jl_task_t*)lst[n];
+            if (mmtk_is_live_object(t)) {
+                jl_task_t *maybe_forwarded = (jl_task_t*)mmtk_get_possibly_forwared(t);
+                live_tasks->items[n] = maybe_forwarded;
+                t = maybe_forwarded;
+                assert(jl_is_task(t));
+                if (t->stkbuf == NULL)
+                    ndel++; // jl_release_task_stack called
+                else
+                    n++;
+            } else {
+                ndel++;
+                void *stkbuf = t->stkbuf;
+                size_t bufsz = t->bufsz;
+                if (stkbuf) {
+                    t->stkbuf = NULL;
+                    _jl_free_stack(ptls2, stkbuf, bufsz);
+                }
+#ifdef _COMPILER_TSAN_ENABLED_
+                if (t->ctx.tsan_state) {
+                    __tsan_destroy_fiber(t->ctx.tsan_state);
+                    t->ctx.tsan_state = NULL;
+                }
+#endif
+            }
+            if (n >= l - ndel)
+                break;
+            void *tmp = lst[n];
+            lst[n] = lst[n + ndel];
+            lst[n + ndel] = tmp;
+        }
+        live_tasks->len -= ndel;
+    }
+}
+
 #define jl_array_data_owner_addr(a) (((jl_value_t**)((char*)a + jl_array_data_owner_offset(jl_array_ndims(a)))))
 
 JL_DLLEXPORT void* get_stackbase(int16_t tid) {
@@ -431,6 +555,7 @@ Julia_Upcalls mmtk_upcalls = (Julia_Upcalls) {
     .mmtk_jl_run_finalizers = mmtk_jl_run_finalizers,
     .jl_throw_out_of_memory_error = jl_throw_out_of_memory_error,
     .sweep_malloced_array = mmtk_sweep_malloced_arrays,
+    .sweep_stack_pools = mmtk_sweep_stack_pools,
     .wait_in_a_safepoint = mmtk_wait_in_a_safepoint,
     .exit_from_safepoint = mmtk_exit_from_safepoint,
     .jl_hrtime = jl_hrtime,
@@ -442,5 +567,6 @@ Julia_Upcalls mmtk_upcalls = (Julia_Upcalls) {
     .arraylist_grow = (void (*)(void*, long unsigned int))arraylist_grow,
     .get_jl_gc_have_pending_finalizers = get_jl_gc_have_pending_finalizers,
     .scan_vm_specific_roots = scan_vm_specific_roots,
+    .update_inlined_array = update_inlined_array,
     .prepare_to_collect = jl_gc_prepare_to_collect,
 };