Supporting moving immix

.github/scripts/ci-build.sh

@@ -12,6 +12,8 @@ fi
 build_type=$1
 # plan to use
 plan=$2
+# moving vs non-moving
+is_moving=$3
 
 # helloworld.jl
 HELLO_WORLD_JL=$BINDING_PATH/.github/scripts/hello_world.jl
@@ -23,9 +25,10 @@ if [ "$build_type" == "release" ]; then
 fi
 
 plan_feature=${plan,,}
+moving_feature=${is_moving,,}
 
 cd $MMTK_JULIA_DIR/mmtk
-cargo build --features $plan_feature $build_args
+cargo build --features $plan_feature,$moving_feature $build_args
 
 cd $JULIA_PATH
 

.github/scripts/ci-test-patching.sh

@@ -24,6 +24,10 @@ declare -a tests_to_skip=(
     # This test checks GC logging
     '@test occursin("GC: pause", read(tmppath, String))' "$JULIA_PATH/test/misc.jl"
 
+    # These tests check for the number of stock GC threads (which we set to 0 with mmtk)
+    '@test (cpu_threads == 1 ? "1" : string(div(cpu_threads, 2))) ==' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test read(`$exename --gcthreads=2 -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test read(`$exename -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
     # This seems to be a regression from upstream when we merge with upstream 43bf2c8.
     # The required string int.jl does not appear in the output even if I test with the stock Julia code.
     # I do not know what is wrong, but at this point, I dont want to spend time on it.

.github/workflows/binding-tests.yml

@@ -6,6 +6,9 @@ on:
       gc_plan:
         required: true
         type: string
+      moving:
+        required: true
+        type: string
 
 jobs:
   build-debug:
@@ -18,7 +21,7 @@ jobs:
           ./.github/scripts/ci-setup.sh
       - name: Build Julia (Debug)
         run: |
-          ./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }}
+          ./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }} ${{ inputs.moving }}
       - name: Style check
         run: |
           ./.github/scripts/ci-style.sh
@@ -37,7 +40,7 @@ jobs:
           ./.github/scripts/ci-test-patching.sh
       - name: Build Julia (Release)
         run: |
-          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }}
+          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} ${{ inputs.moving }}
       - name: Test Julia
         run: |
           ./.github/scripts/ci-test-other.sh
@@ -53,7 +56,7 @@ jobs:
           ./.github/scripts/ci-setup.sh
       - name: Build Julia (Release)
         run: |
-          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }}
+          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} ${{ inputs.moving }}
       - name: Test Julia
         run: |
           ./.github/scripts/ci-test-stdlib.sh
@@ -69,7 +72,7 @@ jobs:
           ./.github/scripts/ci-setup.sh
       - name: Build Julia (Release)
         run: |
-          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }}
+          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} ${{ inputs.moving }}
       - name: Test Julia
         run: |
           ./.github/scripts/ci-test-LinearAlgebra.sh

.github/workflows/ci.yml

@@ -18,6 +18,8 @@ jobs:
       fail-fast: false
       matrix:
         gc_plan: [Immix, StickyImmix]
+        moving: [Default, Non_Moving]
     uses: ./.github/workflows/binding-tests.yml
     with:
       gc_plan: ${{ matrix.gc_plan }}
+      moving: ${{ matrix.moving }}

julia/mmtk_julia.c

@@ -21,6 +21,8 @@ extern void mmtk_store_obj_size_c(void* obj, size_t size);
 extern void jl_gc_free_array(jl_array_t *a);
 extern size_t mmtk_get_obj_size(void* obj);
 extern void jl_rng_split(uint64_t to[JL_RNG_SIZE], uint64_t from[JL_RNG_SIZE]);
+extern void _jl_free_stack(jl_ptls_t ptls, void *stkbuf, size_t bufsz);
+extern void free_stack(void *stkbuf, size_t bufsz);
 extern jl_mutex_t finalizers_lock;
 extern void jl_gc_wait_for_the_world(jl_ptls_t* gc_all_tls_states, int gc_n_threads);
 extern void mmtk_block_thread_for_gc(void);
@@ -114,6 +116,9 @@ static void mmtk_sweep_malloced_arrays(void) JL_NOTSAFEPOINT
                 continue;
             }
             if (mmtk_is_live_object(ma->a)) {
+                // if the array has been forwarded, the reference needs to be updated
+                jl_array_t *maybe_forwarded = (jl_array_t*)mmtk_get_possibly_forwared(ma->a);
+                ma->a = maybe_forwarded;
                 pma = &ma->next;
             }
             else {
@@ -274,6 +279,19 @@ static void add_node_to_roots_buffer(RootsWorkClosure* closure, RootsWorkBuffer*
     }
 }
 
+static void add_node_to_tpinned_roots_buffer(RootsWorkClosure* closure, RootsWorkBuffer* buf, size_t* buf_len, void* root) {
+    if (root == NULL)
+        return;
+
+    buf->ptr[*buf_len] = root;
+    *buf_len += 1;
+    if (*buf_len >= buf->cap) {
+        RootsWorkBuffer new_buf = (closure->report_tpinned_nodes_func)(buf->ptr, *buf_len, buf->cap, closure->data, true);
+        *buf = new_buf;
+        *buf_len = 0;
+    }
+}
+
 void scan_vm_specific_roots(RootsWorkClosure* closure)
 {
     // Create a new buf
@@ -302,10 +320,15 @@ void scan_vm_specific_roots(RootsWorkClosure* closure)
     // constants
     add_node_to_roots_buffer(closure, &buf, &len, jl_emptytuple_type);
     add_node_to_roots_buffer(closure, &buf, &len, cmpswap_names);
-    add_node_to_roots_buffer(closure, &buf, &len, jl_global_roots_table);
+
+    // jl_global_roots_table must be transitively pinned 
+    RootsWorkBuffer tpinned_buf = (closure->report_tpinned_nodes_func)((void**)0, 0, 0, closure->data, true);
+    size_t tpinned_len = 0;
+    add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, jl_global_roots_table);
 
     // Push the result of the work.
     (closure->report_nodes_func)(buf.ptr, len, buf.cap, closure->data, false);
+    (closure->report_tpinned_nodes_func)(tpinned_buf.ptr, tpinned_len, tpinned_buf.cap, closure->data, false);
 }
 
 JL_DLLEXPORT void scan_julia_exc_obj(void* obj_raw, void* closure, ProcessEdgeFn process_edge) {
@@ -349,6 +372,107 @@ JL_DLLEXPORT void scan_julia_exc_obj(void* obj_raw, void* closure, ProcessEdgeFn
     }
 }
 
+// number of stacks to always keep available per pool - from gc-stacks.c
+#define MIN_STACK_MAPPINGS_PER_POOL 5
+
+// if data is inlined inside the array object --- to->data needs to be updated when copying the array
+void update_inlined_array(void* from, void* to) {
+    jl_value_t* jl_from = (jl_value_t*) from;
+    jl_value_t* jl_to = (jl_value_t*) to;
+
+    uintptr_t tag_to = (uintptr_t)jl_typeof(jl_to);
+    jl_datatype_t *vt = (jl_datatype_t*)tag_to;
+
+    if(vt->name == jl_array_typename) {
+        jl_array_t *a = (jl_array_t*)jl_from;
+        jl_array_t *b = (jl_array_t*)jl_to;
+        if (a->flags.how == 0 && mmtk_object_is_managed_by_mmtk(a->data)) { // a is inlined (a->data is an mmtk object)
+            size_t offset_of_data = ((size_t)a->data - a->offset*a->elsize) - (size_t)a;
+            if (offset_of_data > 0 && offset_of_data <= ARRAY_INLINE_NBYTES) {
+                b->data = (void*)((size_t) b + offset_of_data);
+            }
+        }
+    }
+}
+
+// modified sweep_stack_pools from gc-stacks.c
+void mmtk_sweep_stack_pools(void)
+{
+    // Stack sweeping algorithm:
+    //    // deallocate stacks if we have too many sitting around unused
+    //    for (stk in halfof(free_stacks))
+    //        free_stack(stk, pool_sz);
+    //    // then sweep the task stacks
+    //    for (t in live_tasks)
+    //        if (!gc-marked(t))
+    //            stkbuf = t->stkbuf
+    //            bufsz = t->bufsz
+    //            if (stkbuf)
+    //                push(free_stacks[sz], stkbuf)
+    for (int i = 0; i < jl_n_threads; i++) {
+        jl_ptls_t ptls2 = jl_all_tls_states[i];
+
+        // free half of stacks that remain unused since last sweep
+        for (int p = 0; p < JL_N_STACK_POOLS; p++) {
+            arraylist_t *al = &ptls2->heap.free_stacks[p];
+            size_t n_to_free;
+            if (al->len > MIN_STACK_MAPPINGS_PER_POOL) {
+                n_to_free = al->len / 2;
+                if (n_to_free > (al->len - MIN_STACK_MAPPINGS_PER_POOL))
+                    n_to_free = al->len - MIN_STACK_MAPPINGS_PER_POOL;
+            }
+            else {
+                n_to_free = 0;
+            }
+            for (int n = 0; n < n_to_free; n++) {
+                void *stk = arraylist_pop(al);
+                free_stack(stk, pool_sizes[p]);
+            }
+        }
+
+        arraylist_t *live_tasks = &ptls2->heap.live_tasks;
+        size_t n = 0;
+        size_t ndel = 0;
+        size_t l = live_tasks->len;
+        void **lst = live_tasks->items;
+        if (l == 0)
+            continue;
+        while (1) {
+            jl_task_t *t = (jl_task_t*)lst[n];
+            if (mmtk_is_live_object(t)) {
+                jl_task_t *maybe_forwarded = (jl_task_t*)mmtk_get_possibly_forwared(t);
+                live_tasks->items[n] = maybe_forwarded;
+                t = maybe_forwarded;
+                assert(jl_is_task(t));
+                if (t->stkbuf == NULL)
+                    ndel++; // jl_release_task_stack called
+                else
+                    n++;
+            } else {
+                ndel++;
+                void *stkbuf = t->stkbuf;
+                size_t bufsz = t->bufsz;
+                if (stkbuf) {
+                    t->stkbuf = NULL;
+                    _jl_free_stack(ptls2, stkbuf, bufsz);
+                }
+#ifdef _COMPILER_TSAN_ENABLED_
+                if (t->ctx.tsan_state) {
+                    __tsan_destroy_fiber(t->ctx.tsan_state);
+                    t->ctx.tsan_state = NULL;
+                }
+#endif
+            }
+            if (n >= l - ndel)
+                break;
+            void *tmp = lst[n];
+            lst[n] = lst[n + ndel];
+            lst[n + ndel] = tmp;
+        }
+        live_tasks->len -= ndel;
+    }
+}
+
 #define jl_array_data_owner_addr(a) (((jl_value_t**)((char*)a + jl_array_data_owner_offset(jl_array_ndims(a)))))
 
 JL_DLLEXPORT void* get_stackbase(int16_t tid) {
@@ -431,6 +555,7 @@ Julia_Upcalls mmtk_upcalls = (Julia_Upcalls) {
     .mmtk_jl_run_finalizers = mmtk_jl_run_finalizers,
     .jl_throw_out_of_memory_error = jl_throw_out_of_memory_error,
     .sweep_malloced_array = mmtk_sweep_malloced_arrays,
+    .sweep_stack_pools = mmtk_sweep_stack_pools,
     .wait_in_a_safepoint = mmtk_wait_in_a_safepoint,
     .exit_from_safepoint = mmtk_exit_from_safepoint,
     .jl_hrtime = jl_hrtime,
@@ -442,5 +567,6 @@ Julia_Upcalls mmtk_upcalls = (Julia_Upcalls) {
     .arraylist_grow = (void (*)(void*, long unsigned int))arraylist_grow,
     .get_jl_gc_have_pending_finalizers = get_jl_gc_have_pending_finalizers,
     .scan_vm_specific_roots = scan_vm_specific_roots,
+    .update_inlined_array = update_inlined_array,
     .prepare_to_collect = jl_gc_prepare_to_collect,
 };