Supporting moving immix

.github/scripts/ci-test-patching.sh

@@ -24,6 +24,10 @@ declare -a tests_to_skip=(
     # This test checks GC logging
     '@test occursin("GC: pause", read(tmppath, String))' "$JULIA_PATH/test/misc.jl"
 
+    # These tests check for the number of stock GC threads (which we set to 0 with mmtk)
+    '@test (cpu_threads == 1 ? "1" : string(div(cpu_threads, 2))) ==' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test read(`$exename --gcthreads=2 -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test read(`$exename -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
     # This seems to be a regression from upstream when we merge with upstream 43bf2c8.
     # The required string int.jl does not appear in the output even if I test with the stock Julia code.
     # I do not know what is wrong, but at this point, I dont want to spend time on it.

julia/mmtk_julia.c

@@ -21,6 +21,8 @@ extern void mmtk_store_obj_size_c(void* obj, size_t size);
 extern void jl_gc_free_array(jl_array_t *a);
 extern size_t mmtk_get_obj_size(void* obj);
 extern void jl_rng_split(uint64_t to[JL_RNG_SIZE], uint64_t from[JL_RNG_SIZE]);
+extern void _jl_free_stack(jl_ptls_t ptls, void *stkbuf, size_t bufsz);
+extern void free_stack(void *stkbuf, size_t bufsz);
 extern jl_mutex_t finalizers_lock;
 extern void jl_gc_wait_for_the_world(jl_ptls_t* gc_all_tls_states, int gc_n_threads);
 extern void mmtk_block_thread_for_gc(void);
@@ -114,6 +116,9 @@ static void mmtk_sweep_malloced_arrays(void) JL_NOTSAFEPOINT
                 continue;
             }
             if (mmtk_is_live_object(ma->a)) {
+                // if the array has been forwarded, the reference needs to be updated
+                jl_array_t *maybe_forwarded = (jl_array_t*)mmtk_get_possibly_forwared(ma->a);
+                ma->a = maybe_forwarded;
                 pma = &ma->next;
             }
             else {
@@ -274,6 +279,19 @@ static void add_node_to_roots_buffer(RootsWorkClosure* closure, RootsWorkBuffer*
     }
 }
 
+static void add_node_to_tpinned_roots_buffer(RootsWorkClosure* closure, RootsWorkBuffer* buf, size_t* buf_len, void* root) {
+    if (root == NULL)
+        return;
+
+    buf->ptr[*buf_len] = root;
+    *buf_len += 1;
+    if (*buf_len >= buf->cap) {
+        RootsWorkBuffer new_buf = (closure->report_tpinned_nodes_func)(buf->ptr, *buf_len, buf->cap, closure->data, true);
+        *buf = new_buf;
+        *buf_len = 0;
+    }
+}
+
 void scan_vm_specific_roots(RootsWorkClosure* closure)
 {
     // Create a new buf
@@ -302,10 +320,15 @@ void scan_vm_specific_roots(RootsWorkClosure* closure)
     // constants
     add_node_to_roots_buffer(closure, &buf, &len, jl_emptytuple_type);
     add_node_to_roots_buffer(closure, &buf, &len, cmpswap_names);
-    add_node_to_roots_buffer(closure, &buf, &len, jl_global_roots_table);
+
+    // jl_global_roots_table must be transitively pinned 
+    RootsWorkBuffer tpinned_buf = (closure->report_tpinned_nodes_func)((void**)0, 0, 0, closure->data, true);
+    size_t tpinned_len = 0;
+    add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, jl_global_roots_table);
 
     // Push the result of the work.
     (closure->report_nodes_func)(buf.ptr, len, buf.cap, closure->data, false);
+    (closure->report_tpinned_nodes_func)(tpinned_buf.ptr, tpinned_len, tpinned_buf.cap, closure->data, false);
 }
 
 JL_DLLEXPORT void scan_julia_exc_obj(void* obj_raw, void* closure, ProcessEdgeFn process_edge) {
@@ -349,6 +372,107 @@ JL_DLLEXPORT void scan_julia_exc_obj(void* obj_raw, void* closure, ProcessEdgeFn
     }
 }
 
+// number of stacks to always keep available per pool - from gc-stacks.c
+#define MIN_STACK_MAPPINGS_PER_POOL 5
+
+// if data is inlined inside the array object --- to->data needs to be updated when copying the array
+void update_inlined_array(void* from, void* to) {
+    jl_value_t* jl_from = (jl_value_t*) from;
+    jl_value_t* jl_to = (jl_value_t*) to;
+
+    uintptr_t tag_to = (uintptr_t)jl_typeof(jl_to);
+    jl_datatype_t *vt = (jl_datatype_t*)tag_to;
+
+    if(vt->name == jl_array_typename) {
+        jl_array_t *a = (jl_array_t*)jl_from;
+        jl_array_t *b = (jl_array_t*)jl_to;
+        if (a->flags.how == 0 && mmtk_object_is_managed_by_mmtk(a->data)) { // a is inlined (a->data is an mmtk object)
+            size_t offset_of_data = ((size_t)a->data - a->offset*a->elsize) - (size_t)a;
+            if (offset_of_data > 0 && offset_of_data <= ARRAY_INLINE_NBYTES) {
+                b->data = (void*)((size_t) b + offset_of_data);
+            }
+        }
+    }
+}
+
+// modified sweep_stack_pools from gc-stacks.c
+void mmtk_sweep_stack_pools(void)
+{
+    // Stack sweeping algorithm:
+    //    // deallocate stacks if we have too many sitting around unused
+    //    for (stk in halfof(free_stacks))
+    //        free_stack(stk, pool_sz);
+    //    // then sweep the task stacks
+    //    for (t in live_tasks)
+    //        if (!gc-marked(t))
+    //            stkbuf = t->stkbuf
+    //            bufsz = t->bufsz
+    //            if (stkbuf)
+    //                push(free_stacks[sz], stkbuf)
+    for (int i = 0; i < jl_n_threads; i++) {
+        jl_ptls_t ptls2 = jl_all_tls_states[i];
+
+        // free half of stacks that remain unused since last sweep
+        for (int p = 0; p < JL_N_STACK_POOLS; p++) {
+            arraylist_t *al = &ptls2->heap.free_stacks[p];
+            size_t n_to_free;
+            if (al->len > MIN_STACK_MAPPINGS_PER_POOL) {
+                n_to_free = al->len / 2;
+                if (n_to_free > (al->len - MIN_STACK_MAPPINGS_PER_POOL))
+                    n_to_free = al->len - MIN_STACK_MAPPINGS_PER_POOL;
+            }
+            else {
+                n_to_free = 0;
+            }
+            for (int n = 0; n < n_to_free; n++) {
+                void *stk = arraylist_pop(al);
+                free_stack(stk, pool_sizes[p]);
+            }
+        }
+
+        arraylist_t *live_tasks = &ptls2->heap.live_tasks;
+        size_t n = 0;
+        size_t ndel = 0;
+        size_t l = live_tasks->len;
+        void **lst = live_tasks->items;
+        if (l == 0)
+            continue;
+        while (1) {
+            jl_task_t *t = (jl_task_t*)lst[n];
+            if (mmtk_is_live_object(t)) {
+                jl_task_t *maybe_forwarded = (jl_task_t*)mmtk_get_possibly_forwared(t);
+                live_tasks->items[n] = maybe_forwarded;
+                t = maybe_forwarded;
+                assert(jl_is_task(t));
+                if (t->stkbuf == NULL)
+                    ndel++; // jl_release_task_stack called
+                else
+                    n++;
+            } else {
+                ndel++;
+                void *stkbuf = t->stkbuf;
+                size_t bufsz = t->bufsz;
+                if (stkbuf) {
+                    t->stkbuf = NULL;
+                    _jl_free_stack(ptls2, stkbuf, bufsz);
+                }
+#ifdef _COMPILER_TSAN_ENABLED_
+                if (t->ctx.tsan_state) {
+                    __tsan_destroy_fiber(t->ctx.tsan_state);
+                    t->ctx.tsan_state = NULL;
+                }
+#endif
+            }
+            if (n >= l - ndel)
+                break;
+            void *tmp = lst[n];
+            lst[n] = lst[n + ndel];
+            lst[n + ndel] = tmp;
+        }
+        live_tasks->len -= ndel;
+    }
+}
+
 #define jl_array_data_owner_addr(a) (((jl_value_t**)((char*)a + jl_array_data_owner_offset(jl_array_ndims(a)))))
 
 JL_DLLEXPORT void* get_stackbase(int16_t tid) {
@@ -431,6 +555,7 @@ Julia_Upcalls mmtk_upcalls = (Julia_Upcalls) {
     .mmtk_jl_run_finalizers = mmtk_jl_run_finalizers,
     .jl_throw_out_of_memory_error = jl_throw_out_of_memory_error,
     .sweep_malloced_array = mmtk_sweep_malloced_arrays,
+    .sweep_stack_pools = mmtk_sweep_stack_pools,
     .wait_in_a_safepoint = mmtk_wait_in_a_safepoint,
     .exit_from_safepoint = mmtk_exit_from_safepoint,
     .jl_hrtime = jl_hrtime,
@@ -442,5 +567,6 @@ Julia_Upcalls mmtk_upcalls = (Julia_Upcalls) {
     .arraylist_grow = (void (*)(void*, long unsigned int))arraylist_grow,
     .get_jl_gc_have_pending_finalizers = get_jl_gc_have_pending_finalizers,
     .scan_vm_specific_roots = scan_vm_specific_roots,
+    .update_inlined_array = update_inlined_array,
     .prepare_to_collect = jl_gc_prepare_to_collect,
 };

mmtk/Cargo.toml

@@ -9,8 +9,8 @@ edition = "2018"
 # Metadata for the Julia repository
 [package.metadata.julia]
 # Our CI matches the following line and extract mmtk/julia. If this line is updated, please check ci yaml files and make sure it works.
-julia_repo = "https://github.com/mmtk/julia.git"
-julia_version = "5c406d9bb20d76e2298a6101f171cfac491f651c"
+julia_repo = "https://github.com/udesou/julia.git"
+julia_version = "d14df55c99560ba629aae8f01202f84256677ccb"
 
 [lib]
 crate-type = ["cdylib"]
@@ -39,17 +39,19 @@ enum-map = ">=2.1"
 atomic = "0.4.6"
 chrono = "*"
 thread-id = "*"
+memoffset = "*"
 
 # ykstackmaps = { git = "https://github.com/udesou/ykstackmaps.git", branch = "udesou-master", version = "*" }
 
 [features]
-default = ["mmtk/vm_space", "julia_copy_stack"]
+default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning"]
 
 # Plans
 nogc = []
-immix = ["non_moving_immix"]
-stickyimmix = ["non_moving_immix"]
+immix = []
+stickyimmix = ["mmtk/sticky_immix_non_moving_nursery", "mmtk/immix_smaller_block"]
 marksweep = []
+object_pinning = ["mmtk/object_pinning"]
 
 # TODO remove this when we properly support moving
 non_moving_immix = ["mmtk/immix_non_moving", "mmtk/immix_smaller_block"]

mmtk/api/mmtk.h

@@ -22,6 +22,7 @@ typedef struct {
 typedef struct {
     RootsWorkBuffer (*report_edges_func)(void** buf, size_t size, size_t cap, void* data, bool renew);
     RootsWorkBuffer (*report_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew);
+    RootsWorkBuffer (*report_tpinned_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew);
     void* data;
 } RootsWorkClosure;
 
@@ -48,6 +49,8 @@ extern void mmtk_modify_check(void* ref);
 extern int mmtk_object_is_managed_by_mmtk(void* addr);
 extern void mmtk_runtime_panic(void);
 extern void mmtk_unreachable(void);
+extern unsigned char mmtk_pin_object(void* obj);
+extern bool mmtk_is_pinned(void* obj);
 
 extern void mmtk_set_vm_space(void* addr, size_t size);
 extern void mmtk_immortal_region_post_alloc(void* addr, size_t size);
@@ -73,6 +76,7 @@ typedef struct {
     void (* mmtk_jl_run_finalizers) (void* tls);
     void (* jl_throw_out_of_memory_error) (void);
     void (* sweep_malloced_array) (void);
+    void (* sweep_stack_pools) (void);
     void (* wait_in_a_safepoint) (void);
     void (* exit_from_safepoint) (int8_t old_state);
     uint64_t (* jl_hrtime) (void);
@@ -84,6 +88,7 @@ typedef struct {
     void (*arraylist_grow)(void* a, size_t n);
     int* (*get_jl_gc_have_pending_finalizers)(void);
     void (*scan_vm_specific_roots)(RootsWorkClosure* closure);
+    void (*update_inlined_array) (void* from, void* to);
     void (*prepare_to_collect)(void);
 } Julia_Upcalls;
 
@@ -106,6 +111,7 @@ extern void mmtk_run_finalizers_for_obj(void* obj);
 extern void mmtk_run_finalizers(bool at_exit);
 extern void mmtk_gc_poll(void *tls);
 extern void mmtk_julia_copy_stack_check(int copy_stack);
+extern void* mmtk_get_possibly_forwared(void* object);
 
 /**
  * VM Accounting

mmtk/src/api.rs

@@ -517,3 +517,27 @@ pub extern "C" fn mmtk_get_obj_size(obj: ObjectReference) -> usize {
         addr_size.load::<u64>() as usize
     }
 }
+
+#[cfg(feature = "object_pinning")]
+#[no_mangle]
+pub extern "C" fn mmtk_pin_object(object: ObjectReference) -> bool {
+    if !mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) {
+        return false;
+    }
+    memory_manager::pin_object::<JuliaVM>(object)
+}
+
+#[cfg(feature = "object_pinning")]
+#[no_mangle]
+pub extern "C" fn mmtk_unpin_object(object: ObjectReference) -> bool {
+    if !mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) {
+        return false;
+    }
+    memory_manager::unpin_object::<JuliaVM>(object)
+}
+
+#[cfg(feature = "object_pinning")]
+#[no_mangle]
+pub extern "C" fn mmtk_is_pinned(object: ObjectReference) -> bool {
+    memory_manager::is_pinned::<JuliaVM>(object)
+}

mmtk/src/edges.rs

@@ -215,6 +215,13 @@ pub struct RootsWorkClosure {
         factory_ptr: *mut libc::c_void,
         renew: bool,
     ) -> RootsWorkBuffer<ObjectReference>,
+    pub report_tpinned_nodes_func: extern "C" fn(
+        buf: *mut ObjectReference,
+        size: usize,
+        cap: usize,
+        factory_ptr: *mut libc::c_void,
+        renew: bool,
+    ) -> RootsWorkBuffer<ObjectReference>,
     pub factory_ptr: *mut libc::c_void,
 }
 
@@ -262,10 +269,31 @@ impl RootsWorkClosure {
         }
     }
 
+    extern "C" fn report_tpinned_nodes<F: RootsWorkFactory<JuliaVMEdge>>(
+        buf: *mut ObjectReference,
+        size: usize,
+        cap: usize,
+        factory_ptr: *mut libc::c_void,
+        renew: bool,
+    ) -> RootsWorkBuffer<ObjectReference> {
+        if !buf.is_null() {
+            let buf = unsafe { Vec::<ObjectReference>::from_raw_parts(buf, size, cap) };
+            let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) };
+            factory.create_process_tpinning_roots_work(buf);
+        }
+
+        if renew {
+            RootsWorkBuffer::new()
+        } else {
+            RootsWorkBuffer::empty()
+        }
+    }
+
     pub fn from_roots_work_factory<F: RootsWorkFactory<JuliaVMEdge>>(factory: &mut F) -> Self {
         RootsWorkClosure {
             report_edges_func: Self::report_simple_edges::<F>,
             report_nodes_func: Self::report_nodes::<F>,
+            report_tpinned_nodes_func: Self::report_tpinned_nodes::<F>,
             factory_ptr: factory as *mut F as *mut libc::c_void,
         }
     }

mmtk/src/julia_finalizer.rs

@@ -186,6 +186,9 @@ fn mark_finlist<T: ObjectTracer>(list: &mut ArrayListT, start: usize, tracer: &m
     let mut i = start;
     while i < list.len {
         let cur = list.get(i);
+        let cur_i = i;
+        let mut cur_tag: usize = 0;
+
         if cur.is_zero() {
             i += 1;
             continue;
@@ -195,6 +198,7 @@ fn mark_finlist<T: ObjectTracer>(list: &mut ArrayListT, start: usize, tracer: &m
             // Skip next
             i += 1;
             debug_assert!(i < list.len);
+            cur_tag = 1;
             gc_ptr_clear_tag(cur, 1)
         } else {
             ObjectReference::from_raw_address(cur)
@@ -205,10 +209,11 @@ fn mark_finlist<T: ObjectTracer>(list: &mut ArrayListT, start: usize, tracer: &m
         }
 
         let traced = tracer.trace_object(new_obj);
-        debug_assert_eq!(
-            traced, new_obj,
-            "Object is moved -- we need to save the new object back to the finalizer list"
-        );
+        // if object has moved, update the list applying the tag
+        list.set(cur_i, unsafe {
+            Address::from_usize(traced.to_raw_address() | cur_tag)
+        });
+
         i += 1;
     }
 }