From 5a1d35e33af4d32a781f7d2f27bac515b3cb1ec8 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 14 Dec 2022 09:01:17 +0100 Subject: [PATCH] Merge pull request #47741 from JuliaLang/jn/backports-release-1.8-47685 [release 1.8] backport of precompile: serialize the full edges graph (#46920) --- src/dump.c | 1048 +++++++++++++++++++++++-------------- src/gf.c | 11 +- src/jl_exported_funcs.inc | 1 - src/julia.h | 8 +- src/julia_internal.h | 8 +- src/typemap.c | 1 - test/precompile.jl | 58 +- 7 files changed, 720 insertions(+), 415 deletions(-) diff --git a/src/dump.c b/src/dump.c index 920a6981db095..2e2bb2b667919 100644 --- a/src/dump.c +++ b/src/dump.c @@ -104,7 +104,7 @@ static jl_array_t *newly_inferred JL_GLOBALLY_ROOTED; static htable_t queued_method_roots; // inverse of backedges graph (caller=>callees hash) -htable_t edges_map; +jl_array_t *edges_map JL_GLOBALLY_ROOTED; // rooted for the duration of our uses of this // list of requested ccallable signatures static arraylist_t ccallable_list; @@ -157,6 +157,22 @@ uint64_t jl_worklist_key(jl_array_t *worklist) return 0; } +// Use this in a `while` loop to iterate over the backedges in a MethodInstance. +// `*invokesig` will be NULL if the call was made by ordinary dispatch, otherwise +// it will be the signature supplied in an `invoke` call. +// If you don't need `invokesig`, you can set it to NULL on input. +// Initialize iteration with `i = 0`. Returns `i` for the next backedge to be extracted. +static int get_next_edge(jl_array_t *list, int i, jl_value_t** invokesig, jl_method_instance_t **caller) JL_NOTSAFEPOINT +{ + jl_value_t *item = jl_array_ptr_ref(list, i); + if (invokesig != NULL) + *invokesig = NULL; + assert(jl_is_method_instance(item)); + // Not an `invoke` call, it's just the MethodInstance + *caller = (jl_method_instance_t*)item; + return i + 1; +} + // --- serialize --- #define jl_serialize_value(s, v) jl_serialize_value_((s), (jl_value_t*)(v), 0) @@ -241,76 +257,139 @@ static int type_recursively_external(jl_datatype_t *dt) JL_NOTSAFEPOINT return 1; } +static void mark_backedges_in_worklist(jl_method_instance_t *mi, htable_t *visited, int found) +{ + int oldfound = (char*)ptrhash_get(visited, mi) - (char*)HT_NOTFOUND; + if (oldfound < 3) + return; // not in-progress + ptrhash_put(visited, mi, (void*)((char*)HT_NOTFOUND + 1 + found)); +#ifndef NDEBUG + jl_module_t *mod = mi->def.module; + if (jl_is_method(mod)) + mod = ((jl_method_t*)mod)->module; + assert(jl_is_module(mod)); + assert(!mi->precompiled && !module_in_worklist(mod)); + assert(mi->backedges); +#endif + size_t i = 0, n = jl_array_len(mi->backedges); + while (i < n) { + jl_method_instance_t *be; + i = get_next_edge(mi->backedges, i, NULL, &be); + mark_backedges_in_worklist(be, visited, found); + } +} + // When we infer external method instances, ensure they link back to the // package. Otherwise they might be, e.g., for external macros -static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited) +static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, int depth) { - void **bp = ptrhash_bp(visited, mi); - // HT_NOTFOUND: not yet analyzed - // HT_NOTFOUND + 1: doesn't link back - // HT_NOTFOUND + 2: does link back - if (*bp != HT_NOTFOUND) - return (char*)*bp - (char*)HT_NOTFOUND - 1; - *bp = (void*)((char*)HT_NOTFOUND + 1); // preliminarily mark as "not found" - // TODO: this algorithm deals with cycles incorrectly jl_module_t *mod = mi->def.module; if (jl_is_method(mod)) mod = ((jl_method_t*)mod)->module; assert(jl_is_module(mod)); if (mi->precompiled || module_in_worklist(mod)) { - *bp = (void*)((char*)HT_NOTFOUND + 2); // found return 1; } if (!mi->backedges) { return 0; } - size_t i, n = jl_array_len(mi->backedges); - for (i = 0; i < n; i++) { - jl_method_instance_t *be = (jl_method_instance_t*)jl_array_ptr_ref(mi->backedges, i); - if (has_backedge_to_worklist(be, visited)) { - bp = ptrhash_bp(visited, mi); // re-acquire since rehashing might change the location - *bp = (void*)((char*)HT_NOTFOUND + 2); // found - return 1; + void **bp = ptrhash_bp(visited, mi); + // HT_NOTFOUND: not yet analyzed + // HT_NOTFOUND + 1: no link back + // HT_NOTFOUND + 2: does link back + // HT_NOTFOUND + 3 + depth: in-progress + int found = (char*)*bp - (char*)HT_NOTFOUND; + if (found) + return found - 1; + *bp = (void*)((char*)HT_NOTFOUND + 3 + depth); // preliminarily mark as in-progress + size_t i = 0, n = jl_array_len(mi->backedges); + int cycle = 0; + while (i < n) { + jl_method_instance_t *be; + i = get_next_edge(mi->backedges, i, NULL, &be); + int child_found = has_backedge_to_worklist(be, visited, depth + 1); + if (child_found == 1) { + found = 1; + break; + } + else if (child_found >= 2 && child_found - 2 < cycle) { + // record the cycle will resolve at depth "cycle" + cycle = child_found - 2; + assert(cycle); } } - return 0; + if (!found && cycle && cycle != depth) + return cycle + 2; + bp = ptrhash_bp(visited, mi); // re-acquire since rehashing might change the location + *bp = (void*)((char*)HT_NOTFOUND + 1 + found); + if (cycle) { + // If we are the top of the current cycle, now mark all other parts of + // our cycle by re-walking the backedges graph and marking all WIP + // items as found. + // Be careful to only re-walk as far as we had originally scanned above. + // Or if we found a backedge, also mark all of the other parts of the + // cycle as also having an backedge. + n = i; + i = 0; + while (i < n) { + jl_method_instance_t *be; + i = get_next_edge(mi->backedges, i, NULL, &be); + mark_backedges_in_worklist(be, visited, found); + } + } + return found; } // given the list of MethodInstances that were inferred during the // build, select those that are external and have at least one -// relocatable CodeInstance. -static size_t queue_external_mis(jl_array_t *list) +// relocatable CodeInstance and are inferred to be called from the worklist +// or explicitly added by a precompile statement. +// Also prepares external_mis for method_instance_in_queue queries. +static jl_array_t *queue_external_mis(jl_array_t *list) { + if (list == NULL) + return NULL; size_t i, n = 0; htable_t visited; - if (list) { - assert(jl_is_array(list)); - size_t n0 = jl_array_len(list); - htable_new(&visited, n0); - for (i = 0; i < n0; i++) { - jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(list, i); - assert(jl_is_method_instance(mi)); - if (jl_is_method(mi->def.value)) { - jl_method_t *m = mi->def.method; - if (!module_in_worklist(m->module)) { - jl_code_instance_t *ci = mi->cache; - int relocatable = 0; - while (ci) { - relocatable |= ci->relocatability; - ci = ci->next; - } - if (relocatable && ptrhash_get(&external_mis, mi) == HT_NOTFOUND) { - if (has_backedge_to_worklist(mi, &visited)) { - ptrhash_put(&external_mis, mi, mi); - n++; - } + assert(jl_is_array(list)); + size_t n0 = jl_array_len(list); + htable_new(&visited, n0); + for (i = 0; i < n0; i++) { + jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(list, i); + assert(jl_is_method_instance(mi)); + if (jl_is_method(mi->def.value)) { + jl_method_t *m = mi->def.method; + if (!module_in_worklist(m->module)) { + jl_code_instance_t *ci = mi->cache; + while (ci) { + if (ci->max_world == ~(size_t)0 && ci->relocatability && ci->inferred) + break; + ci = jl_atomic_load_relaxed(&ci->next); + } + if (ci && ptrhash_get(&external_mis, mi) == HT_NOTFOUND) { + int found = has_backedge_to_worklist(mi, &visited, 1); + assert(found == 0 || found == 1); + if (found == 1) { + ptrhash_put(&external_mis, mi, ci); + n++; } } } } - htable_free(&visited); } - return n; + htable_free(&visited); + if (n == 0) + return NULL; + jl_array_t *mi_list = jl_alloc_vec_any(n); + n = 0; + for (size_t i = 0; i < external_mis.size; i += 2) { + void *ci = external_mis.table[i+1]; + if (ci != HT_NOTFOUND) { + jl_array_ptr_set(mi_list, n++, (jl_value_t*)ci); + } + } + assert(n == jl_array_len(mi_list)); + return mi_list; } static void jl_serialize_datatype(jl_serializer_state *s, jl_datatype_t *dt) JL_GC_DISABLED @@ -579,19 +658,15 @@ static int jl_serialize_generic(jl_serializer_state *s, jl_value_t *v) JL_GC_DIS } static void jl_serialize_code_instance(jl_serializer_state *s, jl_code_instance_t *codeinst, - int skip_partial_opaque, int internal, - int force) JL_GC_DISABLED + int skip_partial_opaque, int force) JL_GC_DISABLED { - if (internal > 2) { - while (codeinst && !codeinst->relocatability) - codeinst = codeinst->next; - } if (!force && jl_serialize_generic(s, (jl_value_t*)codeinst)) { return; } int validate = 0; - if (codeinst->max_world == ~(size_t)0) + if (codeinst->max_world == ~(size_t)0 && codeinst->inferred) + // TODO: also check if this object is part of the codeinst cache and in edges_map validate = 1; // can check on deserialize if this cache entry is still valid int flags = validate << 0; if (codeinst->invoke == jl_fptr_const_return) @@ -606,7 +681,7 @@ static void jl_serialize_code_instance(jl_serializer_state *s, jl_code_instance_ if (write_ret_type && codeinst->rettype_const && jl_typeis(codeinst->rettype_const, jl_partial_opaque_type)) { if (skip_partial_opaque) { - jl_serialize_code_instance(s, codeinst->next, skip_partial_opaque, internal, 0); + jl_serialize_code_instance(s, codeinst->next, skip_partial_opaque, 0); return; } else { @@ -633,7 +708,7 @@ static void jl_serialize_code_instance(jl_serializer_state *s, jl_code_instance_ jl_serialize_value(s, jl_nothing); } write_uint8(s->s, codeinst->relocatability); - jl_serialize_code_instance(s, codeinst->next, skip_partial_opaque, internal, 0); + jl_serialize_code_instance(s, codeinst->next, skip_partial_opaque, 0); } enum METHOD_SERIALIZATION_MODE { @@ -855,8 +930,6 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li internal = 1; else if (module_in_worklist(mi->def.method->module)) internal = 2; - else if (ptrhash_get(&external_mis, (void*)mi) != HT_NOTFOUND) - internal = 3; write_uint8(s->s, internal); if (!internal) { // also flag this in the backref table as special @@ -875,12 +948,16 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li if (backedges) { // filter backedges to only contain pointers // to items that we will actually store (internal >= 2) - size_t ins, i, l = jl_array_len(backedges); - jl_method_instance_t **b_edges = (jl_method_instance_t**)jl_array_data(backedges); - for (ins = i = 0; i < l; i++) { - jl_method_instance_t *backedge = b_edges[i]; + size_t ins = 0, i = 0, l = jl_array_len(backedges); + jl_value_t **b_edges = (jl_value_t**)jl_array_data(backedges); + jl_value_t *invokeTypes; + jl_method_instance_t *backedge; + while (i < l) { + i = get_next_edge(backedges, i, &invokeTypes, &backedge); if (module_in_worklist(backedge->def.method->module) || method_instance_in_queue(backedge)) { - b_edges[ins++] = backedge; + if (invokeTypes) + b_edges[ins++] = invokeTypes; + b_edges[ins++] = (jl_value_t*)backedge; } } if (ins != l) @@ -890,10 +967,10 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li } jl_serialize_value(s, (jl_value_t*)backedges); jl_serialize_value(s, (jl_value_t*)NULL); //callbacks - jl_serialize_code_instance(s, mi->cache, 1, internal, 0); + jl_serialize_code_instance(s, mi->cache, 1, 0); } else if (jl_is_code_instance(v)) { - jl_serialize_code_instance(s, (jl_code_instance_t*)v, 0, 2, 1); + jl_serialize_code_instance(s, (jl_code_instance_t*)v, 0, 1); } else if (jl_typeis(v, jl_module_type)) { jl_serialize_module(s, (jl_module_t*)v); @@ -1062,28 +1139,12 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li } } -// Used to serialize the external method instances queued in queued_method_roots (from newly_inferred) -static void serialize_htable_keys(jl_serializer_state *s, htable_t *ht, int nitems) -{ - write_int32(s->s, nitems); - void **table = ht->table; - size_t i, n = 0, sz = ht->size; - for (i = 0; i < sz; i += 2) { - if (table[i+1] != HT_NOTFOUND) { - jl_serialize_value(s, (jl_value_t*)table[i]); - n += 1; - } - } - assert(n == nitems); -} - // Create the forward-edge map (caller => callees) // the intent of these functions is to invert the backedges tree // for anything that points to a method not part of the worklist -// or method instances not in the queue // // from MethodTables -static void jl_collect_missing_backedges_to_mod(jl_methtable_t *mt) +static void jl_collect_missing_backedges(jl_methtable_t *mt) { jl_array_t *backedges = mt->backedges; if (backedges) { @@ -1091,26 +1152,39 @@ static void jl_collect_missing_backedges_to_mod(jl_methtable_t *mt) for (i = 1; i < l; i += 2) { jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(backedges, i); jl_value_t *missing_callee = jl_array_ptr_ref(backedges, i - 1); // signature of abstract callee - jl_array_t **edges = (jl_array_t**)ptrhash_bp(&edges_map, (void*)caller); - if (*edges == HT_NOTFOUND) - *edges = jl_alloc_vec_any(0); - jl_array_ptr_1d_push(*edges, missing_callee); + jl_array_t *edges = (jl_array_t*)jl_eqtable_get(edges_map, (jl_value_t*)caller, NULL); + if (edges == NULL) { + edges = jl_alloc_vec_any(0); + JL_GC_PUSH1(&edges); + edges_map = jl_eqtable_put(edges_map, (jl_value_t*)caller, (jl_value_t*)edges, NULL); + JL_GC_POP(); + } + jl_array_ptr_1d_push(edges, NULL); + jl_array_ptr_1d_push(edges, missing_callee); } } } + // from MethodInstances -static void collect_backedges(jl_method_instance_t *callee) JL_GC_DISABLED +static void collect_backedges(jl_method_instance_t *callee, int internal) JL_GC_DISABLED { jl_array_t *backedges = callee->backedges; if (backedges) { - size_t i, l = jl_array_len(backedges); - for (i = 0; i < l; i++) { - jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(backedges, i); - jl_array_t **edges = (jl_array_t**)ptrhash_bp(&edges_map, caller); - if (*edges == HT_NOTFOUND) - *edges = jl_alloc_vec_any(0); - jl_array_ptr_1d_push(*edges, (jl_value_t*)callee); + size_t i = 0, l = jl_array_len(backedges); + while (i < l) { + jl_value_t *invokeTypes; + jl_method_instance_t *caller; + i = get_next_edge(backedges, i, &invokeTypes, &caller); + jl_array_t *edges = (jl_array_t*)jl_eqtable_get(edges_map, (jl_value_t*)caller, NULL); + if (edges == NULL) { + edges = jl_alloc_vec_any(0); + JL_GC_PUSH1(&edges); + edges_map = jl_eqtable_put(edges_map, (jl_value_t*)caller, (jl_value_t*)edges, NULL); + JL_GC_POP(); + } + jl_array_ptr_1d_push(edges, invokeTypes); + jl_array_ptr_1d_push(edges, (jl_value_t*)callee); } } } @@ -1119,24 +1193,21 @@ static void collect_backedges(jl_method_instance_t *callee) JL_GC_DISABLED // For functions owned by modules not on the worklist, call this on each method. // - if the method is owned by a worklist module, add it to the list of things to be // fully serialized -// - otherwise (i.e., if it's an external method), check all of its specializations. -// Collect all external backedges (may be needed later when we invert this list). +// - Collect all backedges (may be needed later when we invert this list). static int jl_collect_methcache_from_mod(jl_typemap_entry_t *ml, void *closure) JL_GC_DISABLED { jl_array_t *s = (jl_array_t*)closure; jl_method_t *m = ml->func.method; - if (module_in_worklist(m->module)) { + if (s && module_in_worklist(m->module)) { jl_array_ptr_1d_push(s, (jl_value_t*)m); jl_array_ptr_1d_push(s, (jl_value_t*)ml->simplesig); } - else { - jl_svec_t *specializations = m->specializations; - size_t i, l = jl_svec_len(specializations); - for (i = 0; i < l; i++) { - jl_method_instance_t *callee = (jl_method_instance_t*)jl_svecref(specializations, i); - if ((jl_value_t*)callee != jl_nothing) - collect_backedges(callee); - } + jl_svec_t *specializations = m->specializations; + size_t i, l = jl_svec_len(specializations); + for (i = 0; i < l; i++) { + jl_method_instance_t *callee = (jl_method_instance_t*)jl_svecref(specializations, i); + if ((jl_value_t*)callee != jl_nothing) + collect_backedges(callee, !s); } return 1; } @@ -1151,8 +1222,8 @@ static void jl_collect_methtable_from_mod(jl_array_t *s, jl_methtable_t *mt) JL_ // Also collect relevant backedges static void jl_collect_extext_methods_from_mod(jl_array_t *s, jl_module_t *m) JL_GC_DISABLED { - if (module_in_worklist(m)) - return; + if (s && module_in_worklist(m)) + s = NULL; // do not collect any methods size_t i; void **table = m->bindings.table; for (i = 1; i < m->bindings.size; i += 2) { @@ -1167,8 +1238,10 @@ static void jl_collect_extext_methods_from_mod(jl_array_t *s, jl_module_t *m) JL if (mt != NULL && (jl_value_t*)mt != jl_nothing && (mt != jl_type_type_mt && mt != jl_nonfunction_mt)) { + assert(mt->module == tn->module); jl_collect_methtable_from_mod(s, mt); - jl_collect_missing_backedges_to_mod(mt); + if (s) + jl_collect_missing_backedges(mt); } } } @@ -1194,103 +1267,154 @@ static void jl_collect_extext_methods_from_mod(jl_array_t *s, jl_module_t *m) JL } } -// flatten the backedge map reachable from caller into callees -static void jl_collect_backedges_to(jl_method_instance_t *caller, htable_t *all_callees) JL_GC_DISABLED +static void jl_record_edges(jl_method_instance_t *caller, arraylist_t *wq, jl_array_t *edges) JL_GC_DISABLED { - if (module_in_worklist(caller->def.method->module) || method_instance_in_queue(caller)) - return; - if (ptrhash_has(&edges_map, caller)) { - jl_array_t **pcallees = (jl_array_t**)ptrhash_bp(&edges_map, (void*)caller), - *callees = *pcallees; - assert(callees != HT_NOTFOUND); - *pcallees = (jl_array_t*) HT_NOTFOUND; + jl_array_t *callees = (jl_array_t*)jl_eqtable_pop(edges_map, (jl_value_t*)caller, NULL, NULL); + if (callees != NULL) { + jl_array_ptr_1d_push(edges, (jl_value_t*)caller); + jl_array_ptr_1d_push(edges, (jl_value_t*)callees); size_t i, l = jl_array_len(callees); - for (i = 0; i < l; i++) { - jl_value_t *c = jl_array_ptr_ref(callees, i); - ptrhash_put(all_callees, c, c); - if (jl_is_method_instance(c)) { - jl_collect_backedges_to((jl_method_instance_t*)c, all_callees); + for (i = 1; i < l; i += 2) { + jl_method_instance_t *c = (jl_method_instance_t*)jl_array_ptr_ref(callees, i); + if (c && jl_is_method_instance(c)) { + arraylist_push(wq, c); } } } } + // Extract `edges` and `ext_targets` from `edges_map` -// This identifies internal->external edges in the call graph, pulling them out for special treatment. -static void jl_collect_backedges( /* edges */ jl_array_t *s, /* ext_targets */ jl_array_t *t) +// `edges` = [caller1, targets_indexes1, ...], the list of methods and their edges +// `ext_targets` is [invokesig1, callee1, matches1, ...], the edges for each target +static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets) { - htable_t all_targets; // target => tgtindex mapping - htable_t all_callees; // MIs called by worklist methods (eff. Set{MethodInstance}) - htable_new(&all_targets, 0); - htable_new(&all_callees, 0); - size_t i; - void **table = edges_map.table; // edges is caller => callees - size_t table_size = edges_map.size; - for (i = 0; i < table_size; i += 2) { - assert(table == edges_map.table && table_size == edges_map.size && + size_t world = jl_atomic_load_acquire(&jl_world_counter); + arraylist_t wq; + arraylist_new(&wq, 0); + void **table = (void**)jl_array_data(edges_map); // edges is caller => callees + size_t table_size = jl_array_len(edges_map); + for (size_t i = 0; i < table_size; i += 2) { + assert(table == jl_array_data(edges_map) && table_size == jl_array_len(edges_map) && "edges_map changed during iteration"); jl_method_instance_t *caller = (jl_method_instance_t*)table[i]; jl_array_t *callees = (jl_array_t*)table[i + 1]; - if (callees == HT_NOTFOUND) + if (callees == NULL) continue; assert(jl_is_method_instance(caller) && jl_is_method(caller->def.method)); - if (module_in_worklist(caller->def.method->module) || method_instance_in_queue(caller)) { - size_t i, l = jl_array_len(callees); - for (i = 0; i < l; i++) { - jl_value_t *c = jl_array_ptr_ref(callees, i); - ptrhash_put(&all_callees, c, c); - if (jl_is_method_instance(c)) { - jl_collect_backedges_to((jl_method_instance_t*)c, &all_callees); - } + if (module_in_worklist(caller->def.method->module) || + method_instance_in_queue(caller)) { + jl_record_edges(caller, &wq, edges); + } + } + while (wq.len) { + jl_method_instance_t *caller = (jl_method_instance_t*)arraylist_pop(&wq); + jl_record_edges(caller, &wq, edges); + } + arraylist_free(&wq); + edges_map = NULL; + htable_t edges_map2; + htable_new(&edges_map2, 0); + htable_t edges_ids; + size_t l = jl_array_len(edges); + htable_new(&edges_ids, l); + for (size_t i = 0; i < l / 2; i++) { + jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, i * 2); + void *target = (void*)((char*)HT_NOTFOUND + i + 1); + ptrhash_put(&edges_ids, (void*)caller, target); + } + // process target list to turn it into a memoized validity table + // and compute the old methods list, ready for serialization + jl_value_t *matches = NULL; + jl_array_t *callee_ids = NULL; + JL_GC_PUSH2(&matches, &callee_ids); + for (size_t i = 0; i < l; i += 2) { + jl_array_t *callees = (jl_array_t*)jl_array_ptr_ref(edges, i + 1); + size_t l = jl_array_len(callees); + callee_ids = jl_alloc_array_1d(jl_array_int32_type, l + 1); + int32_t *idxs = (int32_t*)jl_array_data(callee_ids); + idxs[0] = 0; + size_t nt = 0; + for (size_t j = 0; j < l; j += 2) { + jl_value_t *invokeTypes = jl_array_ptr_ref(callees, j); + jl_value_t *callee = jl_array_ptr_ref(callees, j + 1); + assert(callee && "unsupported edge"); + + if (jl_is_method_instance(callee)) { + jl_methtable_t *mt = jl_method_get_table(((jl_method_instance_t*)callee)->def.method); + if (module_in_worklist(mt->module)) + continue; } - callees = jl_alloc_array_1d(jl_array_int32_type, 0); - void **pc = all_callees.table; - size_t j; - int valid = 1; - for (j = 0; valid && j < all_callees.size; j += 2) { - if (pc[j + 1] != HT_NOTFOUND) { - jl_value_t *callee = (jl_value_t*)pc[j]; - void *target = ptrhash_get(&all_targets, (void*)callee); - if (target == HT_NOTFOUND) { - jl_method_instance_t *callee_mi = (jl_method_instance_t*)callee; - jl_value_t *sig; - if (jl_is_method_instance(callee)) { - sig = callee_mi->specTypes; - } - else { - sig = callee; - } - size_t min_valid = 0; - size_t max_valid = ~(size_t)0; - int ambig = 0; - jl_value_t *matches = jl_matching_methods((jl_tupletype_t*)sig, jl_nothing, -1, 0, jl_atomic_load_acquire(&jl_world_counter), &min_valid, &max_valid, &ambig); - if (matches == jl_false) { - valid = 0; + + // (nullptr, c) => call + // (invokeTypes, c) => invoke + // (nullptr, invokeTypes) => missing call + // (invokeTypes, nullptr) => missing invoke (unused--inferred as Any) + void *target = ptrhash_get(&edges_map2, invokeTypes ? (void*)invokeTypes : (void*)callee); + if (target == HT_NOTFOUND) { + size_t min_valid = 0; + size_t max_valid = ~(size_t)0; + if (invokeTypes) { + jl_methtable_t *mt = jl_method_get_table(((jl_method_instance_t*)callee)->def.method); + if ((jl_value_t*)mt == jl_nothing) { + callee_ids = NULL; // invalid + break; + } + else { + matches = jl_gf_invoke_lookup_worlds(invokeTypes, (jl_value_t*)mt, world, &min_valid, &max_valid); + if (matches == jl_nothing) { + callee_ids = NULL; // invalid break; } - size_t k; - for (k = 0; k < jl_array_len(matches); k++) { - jl_method_match_t *match = (jl_method_match_t *)jl_array_ptr_ref(matches, k); - jl_array_ptr_set(matches, k, match->method); - } - jl_array_ptr_1d_push(t, callee); - jl_array_ptr_1d_push(t, matches); - target = (char*)HT_NOTFOUND + jl_array_len(t) / 2; - ptrhash_put(&all_targets, (void*)callee, target); + matches = (jl_value_t*)((jl_method_match_t*)matches)->method; } - jl_array_grow_end(callees, 1); - ((int32_t*)jl_array_data(callees))[jl_array_len(callees) - 1] = (char*)target - (char*)HT_NOTFOUND - 1; } + else { + jl_value_t *sig; + if (jl_is_method_instance(callee)) + sig = ((jl_method_instance_t*)callee)->specTypes; + else + sig = callee; + int ambig = 0; + matches = jl_matching_methods((jl_tupletype_t*)sig, jl_nothing, + -1, 0, world, &min_valid, &max_valid, &ambig); + if (matches == jl_false) { + callee_ids = NULL; // invalid + break; + } + size_t k; + for (k = 0; k < jl_array_len(matches); k++) { + jl_method_match_t *match = (jl_method_match_t *)jl_array_ptr_ref(matches, k); + jl_array_ptr_set(matches, k, match->method); + } + } + jl_array_ptr_1d_push(ext_targets, invokeTypes); + jl_array_ptr_1d_push(ext_targets, callee); + jl_array_ptr_1d_push(ext_targets, matches); + target = (void*)((char*)HT_NOTFOUND + jl_array_len(ext_targets) / 3); + ptrhash_put(&edges_map2, (void*)callee, target); } - htable_reset(&all_callees, 100); - if (valid) { - jl_array_ptr_1d_push(s, (jl_value_t*)caller); - jl_array_ptr_1d_push(s, (jl_value_t*)callees); + idxs[++nt] = (char*)target - (char*)HT_NOTFOUND - 1; + } + jl_array_ptr_set(edges, i + 1, callee_ids); // swap callees for ids + if (!callee_ids) + continue; + idxs[0] = nt; + // record place of every method in edges + // add method edges to the callee_ids list + for (size_t j = 0; j < l; j += 2) { + jl_value_t *callee = jl_array_ptr_ref(callees, j + 1); + if (callee && jl_is_method_instance(callee)) { + void *target = ptrhash_get(&edges_ids, (void*)callee); + if (target != HT_NOTFOUND) { + idxs[++nt] = (char*)target - (char*)HT_NOTFOUND - 1; + } } } + jl_array_del_end(callee_ids, l - nt); } - htable_free(&all_targets); - htable_free(&all_callees); + JL_GC_POP(); + htable_free(&edges_map2); } // serialize information about all loaded modules @@ -2211,242 +2335,385 @@ static void jl_insert_methods(jl_array_t *list) } } -void remove_code_instance_from_validation(jl_code_instance_t *codeinst) +int remove_code_instance_from_validation(jl_code_instance_t *codeinst) { - ptrhash_remove(&new_code_instance_validate, codeinst); + return ptrhash_remove(&new_code_instance_validate, codeinst); } -static void jl_insert_method_instances(jl_array_t *list) +// verify that these edges intersect with the same methods as before +static jl_array_t *jl_verify_edges(jl_array_t *targets) { - size_t i, l = jl_array_len(list); - // Validate the MethodInstances + size_t world = jl_atomic_load_acquire(&jl_world_counter); + size_t i, l = jl_array_len(targets) / 3; jl_array_t *valids = jl_alloc_array_1d(jl_array_uint8_type, l); memset(jl_array_data(valids), 1, l); - size_t world = jl_atomic_load_acquire(&jl_world_counter); + jl_value_t *loctag = NULL; + jl_value_t *matches = NULL; + JL_GC_PUSH3(&valids, &matches, &loctag); for (i = 0; i < l; i++) { - jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(list, i); - assert(jl_is_method_instance(mi)); - if (jl_is_method(mi->def.method)) { - // Is this still the method we'd be calling? - jl_methtable_t *mt = jl_method_table_for(mi->specTypes); - struct jl_typemap_assoc search = {(jl_value_t*)mi->specTypes, world, NULL, 0, ~(size_t)0}; - jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(mt->defs, &search, /*offs*/0, /*subtype*/1); - if (entry) { - jl_value_t *mworld = entry->func.value; - if (jl_is_method(mworld) && mi->def.method != (jl_method_t*)mworld && jl_type_morespecific(((jl_method_t*)mworld)->sig, mi->def.method->sig)) { - jl_array_uint8_set(valids, i, 0); - invalidate_backedges(&remove_code_instance_from_validation, mi, world, "jl_insert_method_instance"); - // The codeinst of this mi haven't yet been removed - jl_code_instance_t *codeinst = mi->cache; - while (codeinst) { - remove_code_instance_from_validation(codeinst); - codeinst = codeinst->next; - } - if (_jl_debug_method_invalidation) { - jl_array_ptr_1d_push(_jl_debug_method_invalidation, mworld); - jl_array_ptr_1d_push(_jl_debug_method_invalidation, jl_cstr_to_string("jl_method_table_insert")); // GC disabled + jl_value_t *invokesig = jl_array_ptr_ref(targets, i * 3); + jl_value_t *callee = jl_array_ptr_ref(targets, i * 3 + 1); + jl_value_t *expected = jl_array_ptr_ref(targets, i * 3 + 2); + int valid = 1; + size_t min_valid = 0; + size_t max_valid = ~(size_t)0; + if (invokesig) { + assert(callee && "unsupported edge"); + jl_methtable_t *mt = jl_method_get_table(((jl_method_instance_t*)callee)->def.method); + if ((jl_value_t*)mt == jl_nothing) { + valid = 0; + } + else { + matches = jl_gf_invoke_lookup_worlds(invokesig, (jl_value_t*)mt, world, &min_valid, &max_valid); + if (matches == jl_nothing) { + valid = 0; + } + else { + matches = (jl_value_t*)((jl_method_match_t*)matches)->method; + if (matches != expected) { + valid = 0; } } } } - } - // While it's tempting to just remove the invalidated MIs altogether, - // this hurts the ability of SnoopCompile to diagnose problems. - for (i = 0; i < l; i++) { - jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(list, i); - jl_method_instance_t *milive = jl_specializations_get_or_insert(mi); - ptrhash_put(&uniquing_table, mi, milive); // store the association for the 2nd pass - } - // We may need to fix up the backedges for the ones that didn't "go live" - for (i = 0; i < l; i++) { - jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(list, i); - jl_method_instance_t *milive = (jl_method_instance_t*)ptrhash_get(&uniquing_table, mi); - if (milive != mi) { - // A previously-loaded module compiled this method, so the one we deserialized will be dropped. - // But make sure the backedges are copied over. - if (mi->backedges) { - if (!milive->backedges) { - // Copy all the backedges (after looking up the live ones) - size_t j, n = jl_array_len(mi->backedges); - milive->backedges = jl_alloc_vec_any(n); - jl_gc_wb(milive, milive->backedges); - for (j = 0; j < n; j++) { - jl_method_instance_t *be = (jl_method_instance_t*)jl_array_ptr_ref(mi->backedges, j); - jl_method_instance_t *belive = (jl_method_instance_t*)ptrhash_get(&uniquing_table, be); - if (belive == HT_NOTFOUND) - belive = be; - jl_array_ptr_set(milive->backedges, j, belive); - } - } else { - // Copy the missing backedges (this is an O(N^2) algorithm, but many methods have few MethodInstances) - size_t j, k, n = jl_array_len(mi->backedges), nlive = jl_array_len(milive->backedges); - for (j = 0; j < n; j++) { - jl_method_instance_t *be = (jl_method_instance_t*)jl_array_ptr_ref(mi->backedges, j); - jl_method_instance_t *belive = (jl_method_instance_t*)ptrhash_get(&uniquing_table, be); - if (belive == HT_NOTFOUND) - belive = be; - int found = 0; - for (k = 0; k < nlive; k++) { - if (belive == (jl_method_instance_t*)jl_array_ptr_ref(milive->backedges, k)) { - found = 1; - break; - } - } - if (!found) - jl_array_ptr_1d_push(milive->backedges, (jl_value_t*)belive); - } - } + else { + jl_value_t *sig; + if (jl_is_method_instance(callee)) + sig = ((jl_method_instance_t*)callee)->specTypes; + else + sig = callee; + assert(jl_is_array(expected)); + int ambig = 0; + // TODO: possibly need to included ambiguities too (for the optimizer correctness)? + matches = jl_matching_methods((jl_tupletype_t*)sig, jl_nothing, + -1, 0, world, &min_valid, &max_valid, &ambig); + if (matches == jl_false) { + valid = 0; } - // Additionally, if we have CodeInstance(s) and the running CodeInstance is world-limited, transfer it - if (mi->cache && jl_array_uint8_ref(valids, i)) { - if (!milive->cache || milive->cache->max_world < ~(size_t)0) { - jl_code_instance_t *cilive = milive->cache, *ci; - milive->cache = mi->cache; - jl_gc_wb(milive, milive->cache); - ci = mi->cache; - ci->def = milive; - while (ci->next) { - ci = ci->next; - ci->def = milive; + else { + // setdiff!(matches, expected) + size_t j, k, ins = 0; + if (jl_array_len(matches) != jl_array_len(expected)) { + valid = 0; + } + for (k = 0; k < jl_array_len(matches); k++) { + jl_method_t *match = ((jl_method_match_t*)jl_array_ptr_ref(matches, k))->method; + size_t l = jl_array_len(expected); + for (j = 0; j < l; j++) + if (match == (jl_method_t*)jl_array_ptr_ref(expected, j)) + break; + if (j == l) { + // intersection has a new method or a method was + // deleted--this is now probably no good, just invalidate + // everything about it now + valid = 0; + if (!_jl_debug_method_invalidation) + break; + jl_array_ptr_set(matches, ins++, match); } - ci->next = cilive; - jl_gc_wb(ci, ci->next); } + if (!valid && _jl_debug_method_invalidation) + jl_array_del_end((jl_array_t*)matches, jl_array_len(matches) - ins); } } + jl_array_uint8_set(valids, i, valid); + if (!valid && _jl_debug_method_invalidation) { + jl_array_ptr_1d_push(_jl_debug_method_invalidation, invokesig ? (jl_value_t*)invokesig : callee); + loctag = jl_cstr_to_string("insert_backedges_callee"); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); + loctag = jl_box_int32((int32_t)i); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, matches); + } + //jl_static_show((JL_STREAM*)ios_stderr, (jl_value_t*)invokesig); + //jl_static_show((JL_STREAM*)ios_stderr, (jl_value_t*)callee); + //ios_puts(valid ? "valid\n" : "INVALID\n", ios_stderr); } + JL_GC_POP(); + return valids; } -// verify that these edges intersect with the same methods as before -static void jl_verify_edges(jl_array_t *targets, jl_array_t **pvalids) +// Combine all edges relevant to a method into the visited table +void jl_verify_methods(jl_array_t *edges, jl_array_t *valids, htable_t *visited) { - size_t i, l = jl_array_len(targets) / 2; - jl_array_t *valids = jl_alloc_array_1d(jl_array_uint8_type, l); - memset(jl_array_data(valids), 1, l); jl_value_t *loctag = NULL; JL_GC_PUSH1(&loctag); - *pvalids = valids; + size_t i, l = jl_array_len(edges) / 2; + htable_new(visited, l); for (i = 0; i < l; i++) { - jl_value_t *callee = jl_array_ptr_ref(targets, i * 2); - jl_method_instance_t *callee_mi = (jl_method_instance_t*)callee; - jl_value_t *sig; - if (jl_is_method_instance(callee)) { - sig = callee_mi->specTypes; - } - else { - sig = callee; - } - jl_array_t *expected = (jl_array_t*)jl_array_ptr_ref(targets, i * 2 + 1); - assert(jl_is_array(expected)); + jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i); + assert(jl_is_method_instance(caller) && jl_is_method(caller->def.method)); + jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, 2 * i + 1); + assert(jl_typeis((jl_value_t*)callee_ids, jl_array_int32_type)); int valid = 1; - size_t min_valid = 0; - size_t max_valid = ~(size_t)0; - int ambig = 0; - // TODO: possibly need to included ambiguities too (for the optimizer correctness)? - jl_value_t *matches = jl_matching_methods((jl_tupletype_t*)sig, jl_nothing, -1, 0, jl_atomic_load_acquire(&jl_world_counter), &min_valid, &max_valid, &ambig); - if (matches == jl_false || jl_array_len(matches) != jl_array_len(expected)) { + if (callee_ids == NULL) { + // serializing the edges had failed valid = 0; } else { - size_t j, k, l = jl_array_len(expected); - for (k = 0; k < jl_array_len(matches); k++) { - jl_method_match_t *match = (jl_method_match_t*)jl_array_ptr_ref(matches, k); - jl_method_t *m = match->method; - for (j = 0; j < l; j++) { - if (m == (jl_method_t*)jl_array_ptr_ref(expected, j)) - break; - } - if (j == l) { - // intersection has a new method or a method was - // deleted--this is now probably no good, just invalidate - // everything about it now - valid = 0; - break; + int32_t *idxs = (int32_t*)jl_array_data(callee_ids); + size_t j; + for (j = 0; valid && j < idxs[0]; j++) { + int32_t idx = idxs[j + 1]; + valid = jl_array_uint8_ref(valids, idx); + if (!valid && _jl_debug_method_invalidation) { + jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)caller); + loctag = jl_cstr_to_string("verify_methods"); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); + loctag = jl_box_int32((int32_t)idx); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); } } } - jl_array_uint8_set(valids, i, valid); - if (!valid && _jl_debug_method_invalidation) { - jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)callee); - loctag = jl_cstr_to_string("insert_backedges_callee"); - jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); + ptrhash_put(visited, caller, (void*)(((char*)HT_NOTFOUND) + valid + 1)); + //jl_static_show((JL_STREAM*)ios_stderr, (jl_value_t*)caller); + //ios_puts(valid ? "valid\n" : "INVALID\n", ios_stderr); + // HT_NOTFOUND: valid (no invalid edges) + // HT_NOTFOUND + 1: invalid + // HT_NOTFOUND + 2: need to scan + // HT_NOTFOUND + 3 + depth: in-progress + } + JL_GC_POP(); +} + + +// Propagate the result of cycle-resolution to all edges (recursively) +static int mark_edges_in_worklist(jl_array_t *edges, int idx, jl_method_instance_t *cycle, htable_t *visited, int found) +{ + jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, idx * 2); + int oldfound = (char*)ptrhash_get(visited, caller) - (char*)HT_NOTFOUND; + if (oldfound < 3) + return 0; // not in-progress + if (!found) { + ptrhash_remove(visited, (void*)caller); + } + else { + ptrhash_put(visited, (void*)caller, (void*)((char*)HT_NOTFOUND + 1 + found)); + } + jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, idx * 2 + 1); + assert(jl_typeis((jl_value_t*)callee_ids, jl_array_int32_type)); + int32_t *idxs = (int32_t*)jl_array_data(callee_ids); + size_t i, badidx = 0, n = jl_array_len(callee_ids); + for (i = idxs[0] + 1; i < n; i++) { + if (mark_edges_in_worklist(edges, idxs[i], cycle, visited, found) && badidx == 0) + badidx = i - idxs[0]; + } + if (_jl_debug_method_invalidation) { + jl_value_t *loctag = NULL; + JL_GC_PUSH1(&loctag); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)caller); + loctag = jl_cstr_to_string("verify_methods"); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); + jl_method_instance_t *callee = cycle; + if (badidx--) + callee = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * badidx); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)callee); + JL_GC_POP(); + } + return 1; +} + + +// Visit the entire call graph, starting from edges[idx] to determine if that method is valid +static int jl_verify_graph_edge(jl_array_t *edges, int idx, htable_t *visited, int depth) +{ + jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, idx * 2); + assert(jl_is_method_instance(caller) && jl_is_method(caller->def.method)); + int found = (char*)ptrhash_get(visited, (void*)caller) - (char*)HT_NOTFOUND; + if (found == 0) + return 1; // valid + if (found == 1) + return 0; // invalid + if (found != 2) + return found - 1; // depth + found = 0; + ptrhash_put(visited, (void*)caller, (void*)((char*)HT_NOTFOUND + 3 + depth)); // change 2 to in-progress at depth + jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, idx * 2 + 1); + assert(jl_typeis((jl_value_t*)callee_ids, jl_array_int32_type)); + int32_t *idxs = (int32_t*)jl_array_data(callee_ids); + int cycle = 0; + size_t i, n = jl_array_len(callee_ids); + for (i = idxs[0] + 1; i < n; i++) { + int32_t idx = idxs[i]; + int child_found = jl_verify_graph_edge(edges, idx, visited, depth + 1); + if (child_found == 0) { + found = 1; + if (_jl_debug_method_invalidation) { + jl_value_t *loctag = NULL; + JL_GC_PUSH1(&loctag); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)caller); + loctag = jl_cstr_to_string("verify_methods"); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, jl_array_ptr_ref(edges, idx * 2)); + JL_GC_POP(); + } + break; + } + else if (child_found >= 2 && child_found - 2 < cycle) { + // record the cycle will resolve at depth "cycle" + cycle = child_found - 2; + assert(cycle); } } + if (!found) { + if (cycle && cycle != depth) + return cycle + 2; + ptrhash_remove(visited, (void*)caller); + } + else { // found invalid + ptrhash_put(visited, (void*)caller, (void*)((char*)HT_NOTFOUND + 1 + found)); + } + if (cycle) { + // If we are the top of the current cycle, now mark all other parts of + // our cycle by re-walking the backedges graph and marking all WIP + // items as found. + // Be careful to only re-walk as far as we had originally scanned above. + // Or if we found a backedge, also mark all of the other parts of the + // cycle as also having an backedge. + n = i; + for (i = idxs[0] + 1; i < n; i++) { + mark_edges_in_worklist(edges, idxs[i], caller, visited, found); + } + } + return found ? 0 : 1; +} + +// Visit all entries in edges, verify if they are valid +static jl_array_t *jl_verify_graph(jl_array_t *edges, htable_t *visited) +{ + size_t i, n = jl_array_len(edges) / 2; + jl_array_t *valids = jl_alloc_array_1d(jl_array_uint8_type, n); + JL_GC_PUSH1(&valids); + int8_t *valids_data = (int8_t*)jl_array_data(valids); + for (i = 0; i < n; i++) { + valids_data[i] = jl_verify_graph_edge(edges, i, visited, 1); + } JL_GC_POP(); + return valids; } // Restore backedges to external targets -// `targets` is [callee1, matches1, ...], the global set of non-worklist callees of worklist-owned methods. -// `list` = [caller1, targets_indexes1, ...], the list of worklist-owned methods calling external methods. -static void jl_insert_backedges(jl_array_t *list, jl_array_t *targets) +// `edges` = [caller1, targets_indexes1, ...], the list of worklist-owned methods calling external methods. +// `ext_targets` is [invokesig1, callee1, matches1, ...], the global set of non-worklist callees of worklist-owned methods. +static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_array_t *mi_list) { - // map(enable, ((list[i] => targets[list[i + 1] .* 2]) for i in 1:2:length(list) if all(valids[list[i + 1]]))) - size_t i, l = jl_array_len(list); - jl_array_t *valids = NULL; - jl_value_t *loctag = NULL; - JL_GC_PUSH2(&valids, &loctag); - jl_verify_edges(targets, &valids); - for (i = 0; i < l; i += 2) { - jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(list, i); + // determine which CodeInstance objects are still valid in our image + size_t world = jl_atomic_load_acquire(&jl_world_counter); + jl_array_t *valids = jl_verify_edges(ext_targets); + JL_GC_PUSH1(&valids); + htable_t visited; + htable_new(&visited, 0); + jl_verify_methods(edges, valids, &visited); + valids = jl_verify_graph(edges, &visited); + size_t i, l = jl_array_len(edges) / 2; + + // next build a map from external_mis to their CodeInstance for insertion + if (mi_list == NULL) { + htable_reset(&visited, 0); + } + else { + size_t i, l = jl_array_len(mi_list); + htable_reset(&visited, l); + for (i = 0; i < l; i++) { + jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(mi_list, i); + ptrhash_put(&visited, (void*)ci->def, (void*)ci); + } + } + + // next disable any invalid codes, so we do not try to enable them + for (i = 0; i < l; i++) { + jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i); assert(jl_is_method_instance(caller) && jl_is_method(caller->def.method)); - jl_array_t *idxs_array = (jl_array_t*)jl_array_ptr_ref(list, i + 1); - assert(jl_isa((jl_value_t*)idxs_array, jl_array_int32_type)); - int32_t *idxs = (int32_t*)jl_array_data(idxs_array); - int valid = 1; - size_t j; - for (j = 0; valid && j < jl_array_len(idxs_array); j++) { - int32_t idx = idxs[j]; - valid = jl_array_uint8_ref(valids, idx); - } - if (valid) { - // if this callee is still valid, add all the backedges - for (j = 0; j < jl_array_len(idxs_array); j++) { - int32_t idx = idxs[j]; - jl_value_t *callee = jl_array_ptr_ref(targets, idx * 2); - if (jl_is_method_instance(callee)) { - jl_method_instance_add_backedge((jl_method_instance_t*)callee, caller); - } - else { - jl_methtable_t *mt = jl_method_table_for(callee); - // FIXME: rarely, `callee` has an unexpected `Union` signature, - // see https://github.com/JuliaLang/julia/pull/43990#issuecomment-1030329344 - // Fix the issue and turn this back into an `assert((jl_value_t*)mt != jl_nothing)` - // This workaround exposes us to (rare) 265-violations. - if ((jl_value_t*)mt != jl_nothing) - jl_method_table_add_backedge(mt, callee, (jl_value_t*)caller); - } - } - // then enable it + int valid = jl_array_uint8_ref(valids, i); + if (valid) + continue; + void *ci = ptrhash_get(&visited, (void*)caller); + if (ci != HT_NOTFOUND) { + assert(jl_is_code_instance(ci)); + remove_code_instance_from_validation((jl_code_instance_t*)ci); // mark it as handled + } + else { jl_code_instance_t *codeinst = caller->cache; while (codeinst) { - if (ptrhash_get(&new_code_instance_validate, codeinst) != HT_NOTFOUND && codeinst->min_world > 0) - codeinst->max_world = ~(size_t)0; - ptrhash_remove(&new_code_instance_validate, codeinst); // mark it as handled + remove_code_instance_from_validation(codeinst); // should be left invalid codeinst = jl_atomic_load_relaxed(&codeinst->next); } } + } + + // finally enable any applicable new codes + for (i = 0; i < l; i++) { + jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i); + int valid = jl_array_uint8_ref(valids, i); + if (!valid) + continue; + // if this callee is still valid, add all the backedges + jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, 2 * i + 1); + int32_t *idxs = (int32_t*)jl_array_data(callee_ids); + for (size_t j = 0; j < idxs[0]; j++) { + int32_t idx = idxs[j + 1]; + jl_value_t *invokesig = jl_array_ptr_ref(ext_targets, idx * 3); + jl_value_t *callee = jl_array_ptr_ref(ext_targets, idx * 3 + 1); + if (callee && jl_is_method_instance(callee)) { + assert(invokesig == NULL); (void)invokesig; + jl_method_instance_add_backedge((jl_method_instance_t*)callee, caller); + } + else { + jl_value_t *sig = callee == NULL ? invokesig : callee; + jl_methtable_t *mt = jl_method_table_for(sig); + // FIXME: rarely, `callee` has an unexpected `Union` signature, + // see https://github.com/JuliaLang/julia/pull/43990#issuecomment-1030329344 + // Fix the issue and turn this back into an `assert((jl_value_t*)mt != jl_nothing)` + // This workaround exposes us to (rare) 265-violations. + if ((jl_value_t*)mt != jl_nothing) + jl_method_table_add_backedge(mt, sig, (jl_value_t*)caller); + } + } + // then enable it + void *ci = ptrhash_get(&visited, (void*)caller); + if (ci != HT_NOTFOUND) { + // have some new external code to use + assert(jl_is_code_instance(ci)); + jl_code_instance_t *codeinst = (jl_code_instance_t*)ci; + remove_code_instance_from_validation(codeinst); // mark it as handled + assert(codeinst->min_world >= world && codeinst->inferred); + codeinst->max_world = ~(size_t)0; + if (jl_rettype_inferred(caller, world, ~(size_t)0) == jl_nothing) { + jl_mi_cache_insert(caller, codeinst); + } + } else { jl_code_instance_t *codeinst = caller->cache; while (codeinst) { - ptrhash_remove(&new_code_instance_validate, codeinst); // should be left invalid + if (remove_code_instance_from_validation(codeinst)) { // mark it as handled + assert(codeinst->min_world >= world && codeinst->inferred); + codeinst->max_world = ~(size_t)0; + } codeinst = jl_atomic_load_relaxed(&codeinst->next); } - if (_jl_debug_method_invalidation) { - jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)caller); - loctag = jl_cstr_to_string("insert_backedges"); - jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); - } } } + + htable_free(&visited); JL_GC_POP(); } static void validate_new_code_instances(void) { + size_t world = jl_atomic_load_acquire(&jl_world_counter); size_t i; for (i = 0; i < new_code_instance_validate.size; i += 2) { if (new_code_instance_validate.table[i+1] != HT_NOTFOUND) { - ((jl_code_instance_t*)new_code_instance_validate.table[i])->max_world = ~(size_t)0; + jl_code_instance_t *ci = (jl_code_instance_t*)new_code_instance_validate.table[i]; + JL_GC_PROMISE_ROOTED(ci); // TODO: this needs a root (or restructuring to avoid it) + assert(ci->min_world >= world && ci->inferred); + ci->max_world = ~(size_t)0; + jl_method_instance_t *caller = ci->def; + if (jl_rettype_inferred(caller, world, ~(size_t)0) == jl_nothing) { + jl_mi_cache_insert(caller, ci); + } + //jl_static_show((JL_STREAM*)ios_stderr, (jl_value_t*)caller); + //ios_puts("FREE\n", ios_stderr); } } } @@ -2621,13 +2888,18 @@ JL_DLLEXPORT void jl_set_newly_inferred(jl_value_t* _newly_inferred) JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) { JL_TIMING(SAVE_MODULE); + jl_task_t *ct = jl_current_task; ios_t f; - jl_array_t *mod_array = NULL, *udeps = NULL; if (ios_file(&f, fname, 1, 1, 1, 1) == NULL) { jl_printf(JL_STDERR, "Cannot open cache file \"%s\" for writing.\n", fname); return 1; } - JL_GC_PUSH2(&mod_array, &udeps); + + jl_array_t *mod_array = NULL, *udeps = NULL; + jl_array_t *extext_methods = NULL, *mi_list = NULL; + jl_array_t *ext_targets = NULL, *edges = NULL; + JL_GC_PUSH7(&mod_array, &udeps, &extext_methods, &mi_list, &ext_targets, &edges, &edges_map); + mod_array = jl_get_loaded_modules(); // __toplevel__ modules loaded in this session (from Base.loaded_modules_array) assert(jl_precompile_toplevel_module == NULL); jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); @@ -2646,7 +2918,6 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) write_mod_list(&f, mod_array); arraylist_new(&reinit_list, 0); - htable_new(&edges_map, 0); htable_new(&backref_table, 5000); htable_new(&external_mis, newly_inferred ? jl_array_len(newly_inferred) : 0); ptrhash_put(&backref_table, jl_main_module, (char*)HT_NOTFOUND + 1); @@ -2659,15 +2930,14 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) jl_symbol("BITS_PER_LIMB"))) / 8; } - int en = jl_gc_enable(0); // edges map is not gc-safe - jl_array_t *extext_methods = jl_alloc_vec_any(0); // [method1, simplesig1, ...], worklist-owned "extending external" methods added to functions owned by modules outside the worklist - jl_array_t *ext_targets = jl_alloc_vec_any(0); // [callee1, matches1, ...] non-worklist callees of worklist-owned methods - jl_array_t *edges = jl_alloc_vec_any(0); // [caller1, ext_targets_indexes1, ...] for worklist-owned methods calling external methods + jl_gc_enable_finalizers(ct, 0); // make sure we don't run any Julia code concurrently after this point - int n_ext_mis = queue_external_mis(newly_inferred); + // Save the inferred code from newly inferred, external methods + mi_list = queue_external_mis(newly_inferred); - size_t i; - size_t len = jl_array_len(mod_array); + edges_map = jl_alloc_vec_any(0); + extext_methods = jl_alloc_vec_any(0); // [method1, simplesig1, ...], worklist-owned "extending external" methods added to functions owned by modules outside the worklist + size_t i, len = jl_array_len(mod_array); for (i = 0; i < len; i++) { jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(mod_array, i); assert(jl_is_module(m)); @@ -2675,13 +2945,17 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) jl_collect_extext_methods_from_mod(extext_methods, m); } jl_collect_methtable_from_mod(extext_methods, jl_type_type_mt); - jl_collect_missing_backedges_to_mod(jl_type_type_mt); + jl_collect_missing_backedges(jl_type_type_mt); jl_collect_methtable_from_mod(extext_methods, jl_nonfunction_mt); - jl_collect_missing_backedges_to_mod(jl_nonfunction_mt); - - // jl_collect_extext_methods_from_mod and jl_collect_missing_backedges_to_mod accumulate data in edges_map. + jl_collect_missing_backedges(jl_nonfunction_mt); + // jl_collect_extext_methods_from_mod and jl_collect_missing_backedges also accumulate data in edges_map. // Process this to extract `edges` and `ext_targets`. - jl_collect_backedges(edges, ext_targets); + ext_targets = jl_alloc_vec_any(0); // [invokesig1, callee1, matches1, ...] non-worklist callees of worklist-owned methods + // ordinary dispatch: invokesig=NULL, callee is MethodInstance + // `invoke` dispatch: invokesig is signature, callee is MethodInstance + // abstract call: callee is signature + edges = jl_alloc_vec_any(0); // [caller1, ext_targets_indexes1, ...] for worklist-owned methods calling external methods + jl_collect_edges(edges, ext_targets); jl_serializer_state s = { &f, @@ -2690,21 +2964,20 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) }; jl_serialize_value(&s, worklist); // serialize module-owned items (those accessible from the bindings table) jl_serialize_value(&s, extext_methods); // serialize new worklist-owned methods for external functions - serialize_htable_keys(&s, &external_mis, n_ext_mis); // serialize external MethodInstances - // The next two allow us to restore backedges from external "unserialized" (stub-serialized) MethodInstances - // to the ones we serialize here + // The next three allow us to restore code instances, if still valid + jl_serialize_value(&s, mi_list); jl_serialize_value(&s, edges); jl_serialize_value(&s, ext_targets); jl_finalize_serializer(&s); serializer_worklist = NULL; - jl_gc_enable(en); - htable_reset(&edges_map, 0); - htable_reset(&backref_table, 0); - htable_reset(&external_mis, 0); + htable_free(&backref_table); + htable_free(&external_mis); arraylist_free(&reinit_list); + jl_gc_enable_finalizers(ct, 1); // make sure we don't run any Julia code concurrently before this point + // Write the source-text for the dependent files if (udeps) { // Go back and update the source-text position to point to the current position @@ -3084,15 +3357,11 @@ static jl_value_t *_jl_restore_incremental(ios_t *f, jl_array_t *mod_array) }; jl_array_t *restored = (jl_array_t*)jl_deserialize_value(&s, (jl_value_t**)&restored); serializer_worklist = restored; - assert(jl_isa((jl_value_t*)restored, jl_array_any_type)); + assert(jl_typeis((jl_value_t*)restored, jl_array_any_type)); // See explanation in jl_save_incremental for variables of the same names jl_value_t *extext_methods = jl_deserialize_value(&s, &extext_methods); - int i, n_ext_mis = read_int32(s.s); - jl_array_t *mi_list = jl_alloc_vec_any(n_ext_mis); // reload MIs stored by serialize_htable_keys - jl_value_t **midata = (jl_value_t**)jl_array_data(mi_list); - for (i = 0; i < n_ext_mis; i++) - midata[i] = jl_deserialize_value(&s, &(midata[i])); + jl_value_t *mi_list = jl_deserialize_value(&s, &mi_list); // reload MIs stored by queue_external_mis jl_value_t *edges = jl_deserialize_value(&s, &edges); jl_value_t *ext_targets = jl_deserialize_value(&s, &ext_targets); @@ -3106,19 +3375,16 @@ static jl_value_t *_jl_restore_incremental(ios_t *f, jl_array_t *mod_array) jl_insert_methods((jl_array_t*)extext_methods); // hook up extension methods for external generic functions (needs to be after recache types) jl_recache_other(); // make all of the other objects identities correct (needs to be after insert methods) jl_copy_roots(); // copying new roots of external methods (must wait until recaching is complete) - // At this point, the novel specializations in mi_list reference the real method, but they haven't been cached in its specializations - jl_insert_method_instances(mi_list); // insert novel specializations htable_free(&uniquing_table); jl_array_t *init_order = jl_finalize_deserializer(&s, tracee_list); // done with f and s (needs to be after recache) if (init_order == NULL) init_order = (jl_array_t*)jl_an_empty_vec_any; - assert(jl_isa((jl_value_t*)init_order, jl_array_any_type)); + assert(jl_typeis((jl_value_t*)init_order, jl_array_any_type)); - JL_GC_PUSH4(&init_order, &restored, &edges, &ext_targets); + JL_GC_PUSH5(&init_order, &restored, &edges, &ext_targets, &mi_list); jl_gc_enable(en); // subtyping can allocate a lot, not valid before recache-other - jl_insert_backedges((jl_array_t*)edges, (jl_array_t*)ext_targets); // restore external backedges (needs to be last) - + jl_insert_backedges((jl_array_t*)edges, (jl_array_t*)ext_targets, (jl_array_t*)mi_list); // restore external backedges (needs to be last) // check new CodeInstances and validate any that lack external backedges validate_new_code_instances(); diff --git a/src/gf.c b/src/gf.c index 49078f903b708..855a138186b93 100644 --- a/src/gf.c +++ b/src/gf.c @@ -222,8 +222,6 @@ JL_DLLEXPORT jl_code_instance_t* jl_new_codeinst( int32_t const_flags, size_t min_world, size_t max_world, uint32_t ipo_effects, uint32_t effects, jl_value_t *argescapes, uint8_t relocatability); -JL_DLLEXPORT void jl_mi_cache_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, - jl_code_instance_t *ci JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED); jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_args_t fptr) JL_GC_DISABLED { @@ -436,7 +434,10 @@ JL_DLLEXPORT void jl_mi_cache_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMEN JL_GC_PUSH1(&ci); if (jl_is_method(mi->def.method)) JL_LOCK(&mi->def.method->writelock); - ci->next = mi->cache; + jl_code_instance_t *oldci = mi->cache; + ci->next = oldci; + if (oldci) + jl_gc_wb(ci, oldci); jl_atomic_store_release(&mi->cache, ci); jl_gc_wb(mi, ci); if (jl_is_method(mi->def.method)) @@ -1423,6 +1424,7 @@ static void invalidate_method_instance(void (*f)(jl_code_instance_t*), jl_method jl_array_ptr_1d_push(_jl_debug_method_invalidation, boxeddepth); JL_GC_POP(); } + //jl_static_show(JL_STDERR, (jl_value_t*)replaced); if (!jl_is_method(replaced->def.method)) return; // shouldn't happen, but better to be safe JL_LOCK(&replaced->def.method->writelock); @@ -1450,10 +1452,11 @@ static void invalidate_method_instance(void (*f)(jl_code_instance_t*), jl_method } // invalidate cached methods that overlap this definition -void invalidate_backedges(void (*f)(jl_code_instance_t*), jl_method_instance_t *replaced_mi, size_t max_world, const char *why) +static void invalidate_backedges(void (*f)(jl_code_instance_t*), jl_method_instance_t *replaced_mi, size_t max_world, const char *why) { JL_LOCK(&replaced_mi->def.method->writelock); jl_array_t *backedges = replaced_mi->backedges; + //jl_static_show(JL_STDERR, (jl_value_t*)replaced_mi); if (backedges) { // invalidate callers (if any) replaced_mi->backedges = NULL; diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index b81e0ef5b6489..c207fd7518714 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -125,7 +125,6 @@ XX(jl_environ) \ XX(jl_eof_error) \ XX(jl_eqtable_get) \ - XX(jl_eqtable_nextind) \ XX(jl_eqtable_pop) \ XX(jl_eqtable_put) \ XX(jl_errno) \ diff --git a/src/julia.h b/src/julia.h index df637670dca6d..39ca0ae93bbce 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1635,9 +1635,10 @@ STATIC_INLINE jl_function_t *jl_get_function(jl_module_t *m, const char *name) } // eq hash tables -JL_DLLEXPORT jl_array_t *jl_eqtable_put(jl_array_t *h, jl_value_t *key, jl_value_t *val, int *inserted); -JL_DLLEXPORT jl_value_t *jl_eqtable_get(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT; -jl_value_t *jl_eqtable_getkey(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_array_t *jl_eqtable_put(jl_array_t *h JL_ROOTING_ARGUMENT, jl_value_t *key, jl_value_t *val JL_ROOTED_ARGUMENT, int *inserted); +JL_DLLEXPORT jl_value_t *jl_eqtable_get(jl_array_t *h JL_PROPAGATES_ROOT, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_eqtable_pop(jl_array_t *h, jl_value_t *key, jl_value_t *deflt, int *found); +jl_value_t *jl_eqtable_getkey(jl_array_t *h JL_PROPAGATES_ROOT, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT; // system information JL_DLLEXPORT int jl_errno(void) JL_NOTSAFEPOINT; @@ -1814,6 +1815,7 @@ JL_DLLEXPORT jl_value_t *jl_compress_argnames(jl_array_t *syms); JL_DLLEXPORT jl_array_t *jl_uncompress_argnames(jl_value_t *syms); JL_DLLEXPORT jl_value_t *jl_uncompress_argname_n(jl_value_t *syms, size_t i); + JL_DLLEXPORT int jl_is_operator(char *sym); JL_DLLEXPORT int jl_is_unary_operator(char *sym); JL_DLLEXPORT int jl_is_unary_and_binary_operator(char *sym); diff --git a/src/julia_internal.h b/src/julia_internal.h index 65503d472b352..6a9955da48122 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -224,7 +224,6 @@ extern tracer_cb jl_newmeth_tracer; void jl_call_tracer(tracer_cb callback, jl_value_t *tracee); void print_func_loc(JL_STREAM *s, jl_method_t *m); extern jl_array_t *_jl_debug_method_invalidation JL_GLOBALLY_ROOTED; -void invalidate_backedges(void (*f)(jl_code_instance_t*), jl_method_instance_t *replaced_mi, size_t max_world, const char *why); extern JL_DLLEXPORT size_t jl_page_size; extern jl_function_t *jl_typeinf_func; @@ -642,15 +641,18 @@ jl_method_instance_t *jl_method_lookup(jl_value_t **args, size_t nargs, size_t w jl_value_t *jl_gf_invoke_by_method(jl_method_t *method, jl_value_t *gf, jl_value_t **args, size_t nargs); jl_value_t *jl_gf_invoke(jl_value_t *types, jl_value_t *f, jl_value_t **args, size_t nargs); +JL_DLLEXPORT jl_value_t *jl_gf_invoke_lookup_worlds(jl_value_t *types, jl_value_t *mt, size_t world, size_t *min_world, size_t *max_world); JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, jl_value_t *mt, int lim, int include_ambiguous, size_t world, size_t *min_valid, size_t *max_valid, int *ambig); +JL_DLLEXPORT jl_value_t *jl_gf_invoke_lookup_worlds(jl_value_t *types, jl_value_t *mt, size_t world, size_t *min_world, size_t *max_world); + JL_DLLEXPORT jl_datatype_t *jl_first_argument_datatype(jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_argument_datatype(jl_value_t *argt JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_methtable_t *jl_method_table_for( jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_methtable_t *jl_method_get_table( - jl_method_t *method) JL_NOTSAFEPOINT; + jl_method_t *method JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; jl_methtable_t *jl_argument_method_table(jl_value_t *argt JL_PROPAGATES_ROOT); JL_DLLEXPORT int jl_pointer_egal(jl_value_t *t); @@ -874,6 +876,8 @@ JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo( jl_method_instance_t *jl_specializations_get_or_insert(jl_method_instance_t *mi_ins); JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_method_instance_t *caller); JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_value_t *caller); +JL_DLLEXPORT void jl_mi_cache_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, + jl_code_instance_t *ci JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED); uint32_t jl_module_next_counter(jl_module_t *m) JL_NOTSAFEPOINT; jl_tupletype_t *arg_type_tuple(jl_value_t *arg1, jl_value_t **args, size_t nargs); diff --git a/src/typemap.c b/src/typemap.c index dfa8ac67f6abc..2401fd49728c2 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -290,7 +290,6 @@ static jl_typemap_t *mtcache_hash_lookup(jl_array_t *cache JL_PROPAGATES_ROOT, j if (cache == (jl_array_t*)jl_an_empty_vec_any) return (jl_typemap_t*)jl_nothing; jl_typemap_t *ml = (jl_typemap_t*)jl_eqtable_get(cache, ty, jl_nothing); - JL_GC_PROMISE_ROOTED(ml); // clang-sa doesn't trust our JL_PROPAGATES_ROOT claim return ml; } diff --git a/test/precompile.jl b/test/precompile.jl index 072d0ba694699..12634a7512a78 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -630,16 +630,11 @@ precompile_test_harness("code caching") do dir msize = which(size, (Vector{<:Any},)) hasspec = false for i = 1:length(msize.specializations) - if isassigned(msize.specializations, i) - mi = msize.specializations[i] - if isa(mi, Core.MethodInstance) - tt = Base.unwrap_unionall(mi.specTypes) - if tt.parameters[2] == Vector{Cacheb8321416e8a3e2f1.X} - if isdefined(mi, :cache) && isa(mi.cache, Core.CodeInstance) && mi.cache.max_world == typemax(UInt) && mi.cache.inferred !== nothing - hasspec = true - break - end - end + mi = msize.specializations[i] + if isa(mi, Core.MethodInstance) && mi.specTypes == Tuple{typeof(size),Vector{Cacheb8321416e8a3e2f1.X}} + if isdefined(mi, :cache) && isa(mi.cache, Core.CodeInstance) && mi.cache.max_world == typemax(UInt) && mi.cache.inferred !== nothing + hasspec = true + break end end end @@ -659,7 +654,7 @@ precompile_test_harness("code caching") do dir # Check that internal methods and their roots are accounted appropriately minternal = which(M.getelsize, (Vector,)) mi = minternal.specializations[1] - @test Base.unwrap_unionall(mi.specTypes).parameters[2] == Vector{Int32} + @test mi.specTypes == Tuple{typeof(M.getelsize),Vector{Int32}} ci = mi.cache @test ci.relocatability == 1 @test ci.inferred !== nothing @@ -775,7 +770,7 @@ precompile_test_harness("code caching") do dir end end - # Invalidations (this test is adapted from from SnoopCompile) + # Invalidations (this test is adapted from SnoopCompile) function hasvalid(mi, world) isdefined(mi, :cache) || return false ci = mi.cache @@ -805,6 +800,10 @@ precompile_test_harness("code caching") do dir build_stale(37) stale('c') + ## Reporting tests (unrelated to the above) + nbits(::Int8) = 8 + nbits(::Int16) = 16 + end """ ) @@ -823,6 +822,11 @@ precompile_test_harness("code caching") do dir # force precompilation useA() + ## Reporting tests + call_nbits(x::Integer) = $StaleA.nbits(x) + map_nbits() = map(call_nbits, Integer[Int8(1), Int16(1)]) + map_nbits() + end """ ) @@ -844,9 +848,12 @@ precompile_test_harness("code caching") do dir Base.compilecache(Base.PkgId(string(pkg))) end @eval using $StaleA + MA = getfield(@__MODULE__, StaleA) + Base.eval(MA, :(nbits(::UInt8) = 8)) @eval using $StaleC + invalidations = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1) @eval using $StaleB - MA = getfield(@__MODULE__, StaleA) + ccall(:jl_debug_method_invalidation, Any, (Cint,), 0) MB = getfield(@__MODULE__, StaleB) MC = getfield(@__MODULE__, StaleC) world = Base.get_world_counter() @@ -871,6 +878,31 @@ precompile_test_harness("code caching") do dir m = only(methods(MC.call_buildstale)) mi = m.specializations[1] @test hasvalid(mi, world) # was compiled with the new method + + # Reporting test + @test all(i -> isassigned(invalidations, i), eachindex(invalidations)) + m = only(methods(MB.call_nbits)) + for mi in m.specializations + mi === nothing && continue + hv = hasvalid(mi, world) + @test mi.specTypes.parameters[end] === Integer ? !hv : hv + end + + idxs = findall(==("verify_methods"), invalidations) + idxsbits = filter(idxs) do i + mi = invalidations[i-1] + mi.def == m + end + idx = only(idxsbits) + tagbad = invalidations[idx+1] + @test isa(tagbad, Int32) + j = findfirst(==(tagbad), invalidations) + @test invalidations[j-1] == "insert_backedges_callee" + @test isa(invalidations[j-2], Type) + @test isa(invalidations[j+1], Vector{Any}) # [nbits(::UInt8)] + + m = only(methods(MB.map_nbits)) + @test !hasvalid(m.specializations[1], world+1) # insert_backedges invalidations also trigger their backedges end # test --compiled-modules=no command line option