Pin more references and expose some globally rooted symbols (#64)

This PR adds a few more pining for types in the native heap, and exposes globally rooted symbols so we can trace them in the MMTk binding. With this PR, we can run moving Immix without transitively pinning `global_roots_table`. See mmtk/mmtk-julia#170.
* Add `jl_gc_pin`
* Use `jl_gc_pin` to pin `BigFloat` for MPFR
* Rename `PTR_PIN` to `OBJ_PIN`. Additionally add `PTR_PIN` which handles cases for internal pointers.
* Pin objects that are used as key in `jl_codectx_t.global_targets`
* Pin objects that are stored in `jl_codectx_t.PhiNodes`
* Pin objects that are referred to by `jl_cgval_t`
* Disable GC before doing perm alloc in `jl_get_layout` (see mmtk/mmtk-julia#172)
* Pin weak references and the referee (see mmtk/mmtk-julia#176)
* Pin objects that are used in `libuv`'s handle.
* Pin `jl_task_t.completion_future` (see mmtk/mmtk-julia#179)
* Pin all `jl_codeinst_t` objects.
* Pin all `jl_method_instance_t` objects.
* Pin all `jl_module_t` objects.
* Pin all `jl_task_t` objects.
* Expose some `JL_GLOBALLY_ROOTED` symbols, such as `newly_inferred`, `task_done_hook_func`.

base/mpfr.jl

@@ -122,6 +122,7 @@ mutable struct BigFloat <: AbstractFloat
         nb = (nb + Core.sizeof(Limb) - 1) ÷ Core.sizeof(Limb) # align to number of Limb allocations required for this
         #d = Vector{Limb}(undef, nb)
         d = _string_n(nb * Core.sizeof(Limb))
+        ccall(:jl_gc_pin, Cvoid, (Any,), d); # Pin the Julia object
         EXP_NAN = Clong(1) - Clong(typemax(Culong) >> 1)
         return _BigFloat(Clong(precision), one(Cint), EXP_NAN, d) # +NAN
     end

src/aotcompile.cpp

@@ -95,6 +95,7 @@ typedef struct {
     std::vector<GlobalValue*> jl_sysimg_fvars;
     std::vector<GlobalValue*> jl_sysimg_gvars;
     std::map<jl_code_instance_t*, std::tuple<uint32_t, uint32_t>> jl_fvar_map;
+    // This holds references to the heap. Need to be pinned.
     std::vector<void*> jl_value_to_llvm;
     std::vector<jl_code_instance_t*> jl_external_to_llvm;
 } jl_native_code_desc_t;

src/array.c

@@ -243,7 +243,7 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
     // introspect the object to update the a->data field. To avoid doing that and
     // making scan_object much more complex we simply enforce that both owner and
     // buffers are always pinned
-    PTR_PIN(owner);
+    OBJ_PIN(owner);
     a->flags.how = 3;
     a->data = data->data;
     a->flags.isshared = 1;
@@ -296,7 +296,7 @@ JL_DLLEXPORT jl_array_t *jl_string_to_array(jl_value_t *str)
     // introspect the object to update the a->data field. To avoid doing that and
     // making scan_object much more complex we simply enforce that both owner and
     // buffers are always pinned
-    PTR_PIN(str);
+    OBJ_PIN(str);
     a->flags.how = 3;
     a->flags.isshared = 1;
     size_t l = jl_string_len(str);
@@ -695,7 +695,7 @@ static int NOINLINE array_resize_buffer(jl_array_t *a, size_t newlen)
         // introspect the object to update the a->data field. To avoid doing that and
         // making scan_object much more complex we simply enforce that both owner and
         // buffers are always pinned
-        PTR_PIN(s);
+        OBJ_PIN(s);
         jl_array_data_owner(a) = s;
         jl_gc_wb(a, s);
         a->data = jl_string_data(s);

src/ast.c

@@ -722,7 +722,7 @@ static value_t julia_to_list2_noalloc(fl_context_t *fl_ctx, jl_value_t *a, jl_va
 
 static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_valid)
 {
-    PTR_PIN(v);
+    OBJ_PIN(v);
     value_t retval;
     if (julia_to_scm_noalloc1(fl_ctx, v, &retval))
         return retval;

src/builtins.c

@@ -314,7 +314,7 @@ static uintptr_t type_object_id_(jl_value_t *v, jl_varidx_t *env) JL_NOTSAFEPOIN
         }
         // FIXME: Pinning objects that get hashed
         // until we implement address space hashing.
-        PTR_PIN(v);
+        OBJ_PIN(v);
         return inthash((uintptr_t)v);
     }
     if (tv == jl_uniontype_type) {
@@ -364,7 +364,7 @@ static uintptr_t immut_id_(jl_datatype_t *dt, jl_value_t *v, uintptr_t h) JL_NOT
 
         // FIXME: Pinning objects that get hashed
         // until we implement address space hashing.
-        PTR_PIN(v);
+        PTR_PIN(v); // This has to be a pointer pin -- v could be an internal pointer
         // operate element-wise if there are unused bits inside,
         // otherwise just take the whole data block at once
         // a few select pointers (notably symbol) also have special hash values
@@ -424,7 +424,7 @@ static uintptr_t NOINLINE jl_object_id__cold(jl_datatype_t *dt, jl_value_t *v) J
     if (dt->name->mutabl) {
         // FIXME: Pinning objects that get hashed
         // until we implement address space hashing.
-        PTR_PIN(v);
+        OBJ_PIN(v);
         return inthash((uintptr_t)v);
     }
     return immut_id_(dt, v, dt->hash);

src/cgutils.cpp

@@ -295,6 +295,9 @@ static Value *julia_pgv(jl_codectx_t &ctx, const char *cname, void *addr)
     // emit a GlobalVariable for a jl_value_t named "cname"
     // store the name given so we can reuse it (facilitating merging later)
     // so first see if there already is a GlobalVariable for this address
+
+    // This will be stored in the native heap. We need to pin it.
+    OBJ_PIN(addr);
     GlobalVariable* &gv = ctx.global_targets[addr];
     Module *M = jl_Module;
     StringRef localname;
@@ -474,7 +477,7 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p)
     if (!ctx.emission_context.imaging)
         // literal_static_pointer_val will pin p.
         return literal_static_pointer_val(p, ctx.types().T_pjlvalue);
-    PTR_PIN(p);
+    OBJ_PIN(p);
     Value *pgv = literal_pointer_val_slot(ctx, p);
     jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const);
     return ai.decorateInst(maybe_mark_load_dereferenceable(
@@ -491,7 +494,7 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_binding_t *p)
     if (!ctx.emission_context.imaging)
         // literal_static_pointer_val will pin p.
         return literal_static_pointer_val(p, ctx.types().T_pjlvalue);
-    PTR_PIN(p);
+    OBJ_PIN(p);
     // bindings are prefixed with jl_bnd#
     Value *pgv = julia_pgv(ctx, "jl_bnd#", p->name, p->owner, p);
     jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const);

src/codegen.cpp

@@ -1441,6 +1441,8 @@ struct jl_cgval_t {
         promotion_point(nullptr),
         promotion_ssa(-1)
     {
+        // jl_cgval_t could be in the native heap. We have to pin the object references in it.
+        OBJ_PIN(typ);
         assert(TIndex == NULL || TIndex->getType() == getInt8Ty(TIndex->getContext()));
     }
     jl_cgval_t(Value *Vptr, bool isboxed, jl_value_t *typ, Value *tindex, MDNode *tbaa) : // general pointer constructor
@@ -1457,6 +1459,8 @@ struct jl_cgval_t {
     {
         if (Vboxed)
             assert(Vboxed->getType() == JuliaType::get_prjlvalue_ty(Vboxed->getContext()));
+        // jl_cgval_t could be in the native heap. We have to pin the object references in it.
+        OBJ_PIN(typ);
         assert(tbaa != NULL);
         assert(!(isboxed && TIndex != NULL));
         assert(TIndex == NULL || TIndex->getType() == getInt8Ty(TIndex->getContext()));
@@ -1474,6 +1478,9 @@ struct jl_cgval_t {
         promotion_point(nullptr),
         promotion_ssa(-1)
     {
+        // jl_cgval_t could be in the native heap. We have to pin the object references in it.
+        OBJ_PIN(typ);
+        OBJ_PIN(constant);
         assert(jl_is_datatype(typ));
         assert(constant);
     }
@@ -1489,6 +1496,9 @@ struct jl_cgval_t {
         promotion_point(v.promotion_point),
         promotion_ssa(v.promotion_ssa)
     {
+        // jl_cgval_t could be in the native heap. We have to pin the object references in it.
+        OBJ_PIN(typ);
+        OBJ_PIN(constant);
         if (Vboxed)
             assert(Vboxed->getType() == JuliaType::get_prjlvalue_ty(Vboxed->getContext()));
         // this constructor expects we had a badly or equivalently typed version
@@ -1551,13 +1561,15 @@ class jl_codectx_t {
     IRBuilder<> builder;
     jl_codegen_params_t &emission_context;
     llvm::MapVector<jl_code_instance_t*, jl_codegen_call_target_t> call_targets;
+    // This map may hold Julia obj ref in the native heap. We need to pin the void*.
     std::map<void*, GlobalVariable*> &global_targets;
     std::map<std::tuple<jl_code_instance_t*, bool>, GlobalVariable*> &external_calls;
     Function *f = NULL;
     // local var info. globals are not in here.
     std::vector<jl_varinfo_t> slots;
     std::map<int, jl_varinfo_t> phic_slots;
     std::vector<jl_cgval_t> SAvalues;
+    // The vector holds reference to Julia obj ref. We need to pin jl_value_t*.
     std::vector<std::tuple<jl_cgval_t, BasicBlock *, AllocaInst *, PHINode *, jl_value_t *>> PhiNodes;
     std::vector<bool> ssavalue_assigned;
     std::vector<int> ssavalue_usecount;
@@ -4851,6 +4863,7 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r)
                 maybe_bitcast(ctx, decay_derived(ctx, phi), getInt8PtrTy(ctx.builder.getContext())));
             jl_cgval_t val = mark_julia_slot(ptr, phiType, Tindex_phi, ctx.tbaa().tbaa_stack); // XXX: this TBAA is wrong for ptr_phi
             val.Vboxed = ptr_phi;
+            OBJ_PIN(r);
             ctx.PhiNodes.push_back(std::make_tuple(val, BB, dest, ptr_phi, r));
             ctx.SAvalues.at(idx) = val;
             ctx.ssavalue_assigned.at(idx) = true;
@@ -4860,6 +4873,7 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r)
             PHINode *Tindex_phi = PHINode::Create(getInt8Ty(ctx.builder.getContext()), jl_array_len(edges), "tindex_phi");
             BB->getInstList().insert(InsertPt, Tindex_phi);
             jl_cgval_t val = mark_julia_slot(NULL, phiType, Tindex_phi, ctx.tbaa().tbaa_stack);
+            OBJ_PIN(r);
             ctx.PhiNodes.push_back(std::make_tuple(val, BB, dest, (PHINode*)NULL, r));
             ctx.SAvalues.at(idx) = val;
             ctx.ssavalue_assigned.at(idx) = true;
@@ -4895,6 +4909,7 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r)
         BB->getInstList().insert(InsertPt, value_phi);
         slot = mark_julia_type(ctx, value_phi, isboxed, phiType);
     }
+    OBJ_PIN(r);
     ctx.PhiNodes.push_back(std::make_tuple(slot, BB, dest, value_phi, r));
     ctx.SAvalues.at(idx) = slot;
     ctx.ssavalue_assigned.at(idx) = true;

src/datatype.c

@@ -67,7 +67,7 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu
                                     jl_typename_type);
     // Typenames should be pinned since they are used as metadata, and are
     // read during scan_object
-    PTR_PIN(tn);
+    OBJ_PIN(tn);
     tn->name = name;
     tn->module = module;
     tn->wrapper = NULL;
@@ -101,7 +101,7 @@ jl_datatype_t *jl_new_uninitialized_datatype(void)
     jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc(ct->ptls, sizeof(jl_datatype_t), jl_datatype_type);
     // Types should be pinned since they are used as metadata, and are
     // read during scan_object
-    PTR_PIN(t);
+    OBJ_PIN(t);
     t->hash = 0;
     t->hasfreetypevars = 0;
     t->isdispatchtuple = 0;
@@ -643,7 +643,14 @@ void jl_compute_field_offsets(jl_datatype_t *st)
             }
         }
         assert(ptr_i == npointers);
+        // Work around mmtk-julia #172: Julia assumes perm alloc won't trigger GC.
+#ifdef MMTK_GC
+        int en = jl_gc_enable(0);
+#endif
         st->layout = jl_get_layout(sz, nfields, npointers, alignm, haspadding, desc, pointers);
+#ifdef MMTK_GC
+        jl_gc_enable(en);
+#endif
         if (should_malloc) {
             free(desc);
             if (npointers)
@@ -801,7 +808,14 @@ JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t *
     // and we easily have a free bit for it in the DataType flags
     bt->isprimitivetype = 1;
     bt->isbitstype = (parameters == jl_emptysvec);
+    // Work around mmtk-julia #172: Julia assumes perm alloc won't trigger GC.
+#ifdef MMTK_GC
+    int en = jl_gc_enable(0);
+#endif
     bt->layout = jl_get_layout(nbytes, 0, 0, alignm, 0, NULL, NULL);
+#ifdef MMTK_GC
+    jl_gc_enable(en);
+#endif
     bt->instance = NULL;
     return bt;
 }

src/gc-common.c

@@ -859,6 +859,11 @@ JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref_th(jl_ptls_t ptls,
     jl_weakref_t *wr = (jl_weakref_t*)jl_gc_alloc(ptls, sizeof(void*),
                                                   jl_weakref_type);
     wr->value = value;  // NOTE: wb not needed here
+    // We need this pin. The weak refs are stored in an arraylist.
+    OBJ_PIN(wr);
+    // This pin is not necessary. We can deal with object forwarding in weak ref processing.
+    // Pinning is simpler.
+    OBJ_PIN(value);
     small_arraylist_push(&ptls->heap.weak_refs, wr);
     return wr;
 }

src/gc.c

@@ -3619,6 +3619,11 @@ JL_DLLEXPORT void jl_gc_wb2_slow(const void *parent, const void* ptr) JL_NOTSAFE
     jl_unreachable();
 }
 
+JL_DLLEXPORT void jl_gc_pin(jl_value_t* obj)
+{
+    // Nothing needs to be done
+}
+
 #ifdef __cplusplus
 }
 #endif

src/gc.h

@@ -744,6 +744,8 @@ void gc_stats_big_obj(void);
 #define gc_stats_big_obj()
 #endif
 
+JL_DLLEXPORT void jl_gc_pin(jl_value_t* obj);
+
 // For debugging
 void gc_count_pool(void);
 

src/gf.c

@@ -430,6 +430,8 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst(
     jl_atomic_store_relaxed(&codeinst->purity_bits, effects);
     codeinst->argescapes = argescapes;
     codeinst->relocatability = relocatability;
+    // Pin codeinst, as they are referenced by vectors and maps in _jl_codegen_params_t
+    OBJ_PIN(codeinst);
     return codeinst;
 }
 

src/jitlayers.h

@@ -175,6 +175,7 @@ typedef struct _jl_codegen_params_t {
     typedef StringMap<GlobalVariable*> SymMapGV;
     // outputs
     std::vector<std::pair<jl_code_instance_t*, jl_codegen_call_target_t>> workqueue;
+    // This may hold references to Julia heap. void* needs to be pinned.
     std::map<void*, GlobalVariable*> globals;
     std::map<std::tuple<jl_code_instance_t*,bool>, GlobalVariable*> external_fns;
     std::map<jl_datatype_t*, DIType*> ditypes;
@@ -242,7 +243,7 @@ void add_named_global(StringRef name, void *addr);
 
 static inline Constant *literal_static_pointer_val(const void *p, Type *T)
 {
-    PTR_PIN((void*)p);
+    PTR_PIN((void*)p); // This may point to non-mmtk heap memory.
     // this function will emit a static pointer into the generated code
     // the generated code will only be valid during the current session,
     // and thus, this should typically be avoided in new API's

src/jl_exported_funcs.inc

@@ -210,6 +210,7 @@
     XX(jl_gc_set_max_memory) \
     XX(jl_gc_sync_total_bytes) \
     XX(jl_gc_total_hrtime) \
+    XX(jl_gc_pin) \
     XX(jl_gdblookup) \
     XX(jl_generating_output) \
     XX(jl_generic_function_def) \

src/jl_uv.c

@@ -275,11 +275,13 @@ JL_DLLEXPORT void jl_forceclose_uv(uv_handle_t *handle)
 JL_DLLEXPORT void jl_uv_associate_julia_struct(uv_handle_t *handle,
                                                jl_value_t *data)
 {
+    OBJ_PIN(data);
     handle->data = data;
 }
 
 JL_DLLEXPORT void jl_uv_disassociate_julia_struct(uv_handle_t *handle)
 {
+    // TODO: unpin here -- we need to implement pin count before we can unpin objects.
     handle->data = NULL;
 }
 

src/julia.h

@@ -27,6 +27,10 @@ extern "C" {
 #endif
 
 extern int mmtk_object_is_managed_by_mmtk(void* addr);
+// Pointers include off-heap pointers, internal pointers, and normal object reference
+extern unsigned char mmtk_pin_pointer(void* ptr);
+extern unsigned char mmtk_unpin_pointer(void* ptr);
+// Object can only be in-heap object references.
 extern unsigned char mmtk_pin_object(void* obj);
 extern unsigned char mmtk_unpin_object(void* obj);
 #ifdef __clang_gcanalyzer__
@@ -36,8 +40,9 @@ extern void PTRHASH_UNPIN(void* key) JL_NOTSAFEPOINT;
 // FIXME: Pinning objects that get hashed in the ptrhash table
 // until we implement address space hashing.
 #ifdef MMTK_GC
-#define PTRHASH_PIN(key) mmtk_pin_object(key);
-#define PTRHASH_UNPIN(key) mmtk_unpin_object(key);
+// Can we use pin objects?
+#define PTRHASH_PIN(key) if (key) mmtk_pin_pointer(key);
+#define PTRHASH_UNPIN(key) if (key) mmtk_unpin_pointer(key);
 #else
 #define PTRHASH_PIN(key)
 #define PTRHASH_UNPIN(key)

src/julia_internal.h

@@ -8,20 +8,32 @@ extern "C" {
 #endif
 
 extern int mmtk_object_is_managed_by_mmtk(void* addr);
+// Pointers include off-heap pointers, internal pointers, and normal object reference
+extern unsigned char mmtk_pin_pointer(void* ptr);
+extern unsigned char mmtk_unpin_pointer(void* ptr);
+extern unsigned char mmtk_is_pointer_pinned(void* ptr);
+// Object can only be in-heap object references.
 extern unsigned char mmtk_pin_object(void* obj);
 extern unsigned char mmtk_unpin_object(void* obj);
+extern unsigned char mmtk_is_object_pinned(void* obj);
 
 #ifdef __clang_gcanalyzer__
+extern void OBJ_PIN(void* key) JL_NOTSAFEPOINT;
+extern void OBJ_UNPIN(void* key) JL_NOTSAFEPOINT;
 extern void PTR_PIN(void* key) JL_NOTSAFEPOINT;
 extern void PTR_UNPIN(void* key) JL_NOTSAFEPOINT;
 #else
 
 #ifdef MMTK_GC
-#define PTR_PIN(key) mmtk_pin_object(key);
-#define PTR_UNPIN(key) mmtk_unpin_object(key);
+#define PTR_PIN(key) if (key) mmtk_pin_pointer(key);
+#define PTR_UNPIN(key) if (key) mmtk_unpin_pointer(key);
+#define OBJ_PIN(key) if (key) mmtk_pin_object(key);
+#define OBJ_UNPIN(key) if (key) mmtk_unpin_object(key);
 #else
 #define PTR_PIN(key)
 #define PTR_UNPIN(key)
+#define OBJ_PIN(key)
+#define OBJ_UNPIN(key)
 #endif
 
 #endif
@@ -588,7 +600,7 @@ STATIC_INLINE jl_gc_tracked_buffer_t *jl_gc_alloc_buf(jl_ptls_t ptls, size_t sz)
     // introspect the object to update the a->data field. To avoid doing that and
     // making scan_object much more complex we simply enforce that both owner and
     // buffers are always pinned
-    PTR_PIN(buf);
+    OBJ_PIN(buf);
     return buf;
 }
 

src/method.c

@@ -449,6 +449,8 @@ JL_DLLEXPORT jl_method_instance_t *jl_new_method_instance_uninit(void)
     li->inInference = 0;
     li->cache_with_orig = 0;
     li->precompiled = 0;
+    // jl_method_instance_t needs to be pinned, as it is referenced in a map in JITDebugInfoRegistry
+    OBJ_PIN(li);
     return li;
 }