diff --git a/src/Makefile b/src/Makefile index 08fb816d302a03..334a9baa6ade8d 100644 --- a/src/Makefile +++ b/src/Makefile @@ -44,7 +44,7 @@ SRCS := \ jltypes gf typemap smallintset ast builtins module interpreter symbol \ dlload sys init task array staticdata toplevel jl_uv datatype \ simplevector runtime_intrinsics precompile jloptions mtarraylist \ - threading partr stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler method \ + threading partr stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler gc-page-profiler method \ jlapi signal-handling safepoint timing subtype rtutils gc-heap-snapshot \ crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall diff --git a/src/gc-page-profiler.c b/src/gc-page-profiler.c new file mode 100644 index 00000000000000..5af1c3d014770d --- /dev/null +++ b/src/gc-page-profiler.c @@ -0,0 +1,167 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include "gc-page-profiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// whether page profiling is enabled +int page_profile_enabled; +// number of pages written +size_t page_profile_pages_written; +// stream to write page profile to +ios_t *page_profile_stream; +// mutex for page profile +uv_mutex_t page_profile_lock; + +gc_page_profiler_serializer_t gc_page_serializer_create(void) JL_NOTSAFEPOINT +{ + gc_page_profiler_serializer_t serializer; + if (__unlikely(page_profile_enabled)) { + arraylist_new(&serializer.typestrs, GC_PAGE_SZ); + } + else { + serializer.typestrs.len = 0; + } + return serializer; +} + +void gc_page_serializer_init(gc_page_profiler_serializer_t *serializer, + jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + serializer->typestrs.len = 0; + serializer->data = (char *)pg->data; + serializer->osize = pg->osize; + } +} + +void gc_page_serializer_destroy(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + arraylist_free(&serializer->typestrs); + } +} + +void gc_page_serializer_write(gc_page_profiler_serializer_t *serializer, + const char *str) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + arraylist_push(&serializer->typestrs, (void *)str); + } +} + +void gc_enable_page_profile(void) JL_NOTSAFEPOINT +{ + page_profile_enabled = 1; +} + +void gc_disable_page_profile(void) JL_NOTSAFEPOINT +{ + page_profile_enabled = 0; +} + +int gc_page_profile_is_enabled(void) JL_NOTSAFEPOINT +{ + return page_profile_enabled; +} + +void gc_page_profile_write_preamble(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + char str[GC_TYPE_STR_MAXLEN]; + snprintf(str, GC_TYPE_STR_MAXLEN, + "{\"address\": \"%p\",\"object_size\": %d,\"objects\": [", + serializer->data, serializer->osize); + ios_write(page_profile_stream, str, strlen(str)); + } +} + +void gc_page_profile_write_epilogue(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + const char *str = "]}"; + ios_write(page_profile_stream, str, strlen(str)); + } +} + +void gc_page_profile_write_comma(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + // write comma if not first page + if (page_profile_pages_written > 0) { + const char *str = ","; + ios_write(page_profile_stream, str, strlen(str)); + } + } +} + +void gc_page_profile_write_to_file(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + // write to file + uv_mutex_lock(&page_profile_lock); + gc_page_profile_write_comma(serializer); + gc_page_profile_write_preamble(serializer); + char str[GC_TYPE_STR_MAXLEN]; + for (size_t i = 0; i < serializer->typestrs.len; i++) { + const char *name = (const char *)serializer->typestrs.items[i]; + if (name == GC_SERIALIZER_EMPTY) { + snprintf(str, GC_TYPE_STR_MAXLEN, "\"empty\","); + } + else if (name == GC_SERIALIZER_GARBAGE) { + snprintf(str, GC_TYPE_STR_MAXLEN, "\"garbage\","); + } + else { + snprintf(str, GC_TYPE_STR_MAXLEN, "\"%s\",", name); + } + // remove trailing comma for last element + if (i == serializer->typestrs.len - 1) { + str[strlen(str) - 1] = '\0'; + } + ios_write(page_profile_stream, str, strlen(str)); + } + gc_page_profile_write_epilogue(serializer); + page_profile_pages_written++; + uv_mutex_unlock(&page_profile_lock); + } +} + +void gc_page_profile_write_json_preamble(ios_t *stream) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + uv_mutex_lock(&page_profile_lock); + const char *str = "{\"pages\": ["; + ios_write(stream, str, strlen(str)); + uv_mutex_unlock(&page_profile_lock); + } +} + +void gc_page_profile_write_json_epilogue(ios_t *stream) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + uv_mutex_lock(&page_profile_lock); + const char *str = "]}"; + ios_write(stream, str, strlen(str)); + uv_mutex_unlock(&page_profile_lock); + } +} + +JL_DLLEXPORT void jl_gc_take_page_profile(ios_t *stream) +{ + gc_enable_page_profile(); + page_profile_pages_written = 0; + page_profile_stream = stream; + gc_page_profile_write_json_preamble(stream); + jl_gc_collect(JL_GC_FULL); + gc_page_profile_write_json_epilogue(stream); + gc_disable_page_profile(); +} + +#ifdef __cplusplus +} +#endif diff --git a/src/gc-page-profiler.h b/src/gc-page-profiler.h new file mode 100644 index 00000000000000..b103e23905ba52 --- /dev/null +++ b/src/gc-page-profiler.h @@ -0,0 +1,63 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#ifndef GC_PAGE_PROFILER_H +#define GC_PAGE_PROFILER_H + +#include "gc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define GC_TYPE_STR_MAXLEN (512) + +typedef struct { + arraylist_t typestrs; + char *data; + int osize; +} gc_page_profiler_serializer_t; + +// mutex for page profile +extern uv_mutex_t page_profile_lock; + +// Serializer functions +gc_page_profiler_serializer_t gc_page_serializer_create(void) JL_NOTSAFEPOINT; +void gc_page_serializer_init(gc_page_profiler_serializer_t *serializer, jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT; +void gc_page_serializer_destroy(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; +void gc_page_serializer_write(gc_page_profiler_serializer_t *serializer, const char *str) JL_NOTSAFEPOINT; +// Page profile functions +#define GC_SERIALIZER_EMPTY ((const char *)0x1) +#define GC_SERIALIZER_GARBAGE ((const char *)0x2) +STATIC_INLINE void gc_page_profile_write_empty_page(gc_page_profiler_serializer_t *serializer, + int enabled) JL_NOTSAFEPOINT +{ + if (__unlikely(enabled)) { + gc_page_serializer_write(serializer, GC_SERIALIZER_EMPTY); + } +} +STATIC_INLINE void gc_page_profile_write_garbage(gc_page_profiler_serializer_t *serializer, + int enabled) JL_NOTSAFEPOINT +{ + if (__unlikely(enabled)) { + gc_page_serializer_write(serializer, GC_SERIALIZER_GARBAGE); + } +} +STATIC_INLINE void gc_page_profile_write_live_obj(gc_page_profiler_serializer_t *serializer, + jl_taggedvalue_t *v, + int enabled) JL_NOTSAFEPOINT +{ + if (__unlikely(enabled)) { + const char *name = jl_typeof_str(jl_valueof(v)); + gc_page_serializer_write(serializer, name); + } +} +void gc_enable_page_profile(void) JL_NOTSAFEPOINT; +void gc_disable_page_profile(void) JL_NOTSAFEPOINT; +int gc_page_profile_is_enabled(void) JL_NOTSAFEPOINT; +void gc_page_profile_write_to_file(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; + +#ifdef __cplusplus +} +#endif + +#endif // GC_PAGE_PROFILER_H diff --git a/src/gc.c b/src/gc.c index b34b37dc79faeb..dda869c4173294 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1,6 +1,7 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "gc.h" +#include "gc-page-profiler.h" #include "julia.h" #include "julia_gcext.h" #include "julia_assert.h" @@ -1421,11 +1422,11 @@ STATIC_INLINE void gc_dump_page_utilization_data(void) JL_NOTSAFEPOINT int64_t buffered_pages = 0; // Returns pointer to terminal pointer of list rooted at *pfl. -static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered, +static void gc_sweep_page(gc_page_profiler_serializer_t *s, jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered, jl_gc_pagemeta_t *pg, int osize) JL_NOTSAFEPOINT { char *data = pg->data; - jl_taggedvalue_t *v = (jl_taggedvalue_t*)(data + GC_PAGE_OFFSET); + jl_taggedvalue_t *v0 = (jl_taggedvalue_t*)(data + GC_PAGE_OFFSET); char *lim = data + GC_PAGE_SZ - osize; char *lim_newpages = data + GC_PAGE_SZ; if (gc_page_data((char*)p->newpages - 1) == data) { @@ -1433,6 +1434,9 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag } size_t old_nfree = pg->nfree; size_t nfree; + // avoid loading a global variable in the hot path + int page_profile_enabled = gc_page_profile_is_enabled(); + gc_page_serializer_init(s, pg); int re_use_page = 1; int keep_as_local_buffer = 0; @@ -1452,6 +1456,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag } #endif nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / osize; + gc_page_profile_write_empty_page(s, page_profile_enabled); goto done; } // For quick sweep, we might be able to skip the page if the page doesn't @@ -1461,12 +1466,13 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag if (!prev_sweep_full || pg->prev_nold == pg->nold) { freedall = 0; nfree = pg->nfree; + gc_page_profile_write_empty_page(s, page_profile_enabled); goto done; } } pg_skpd = 0; - { // scope to avoid clang goto errors + { // scope to avoid clang goto errors int has_marked = 0; int has_young = 0; int16_t prev_nold = 0; @@ -1474,6 +1480,22 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag jl_taggedvalue_t *fl = NULL; jl_taggedvalue_t **pfl = &fl; jl_taggedvalue_t **pfl_begin = NULL; + // collect page profile + jl_taggedvalue_t *v = v0; + if (page_profile_enabled) { + while ((char*)v <= lim) { + int bits = v->bits.gc; + if (!gc_marked(bits) || (char*)v >= lim_newpages) { + gc_page_profile_write_garbage(s, page_profile_enabled); + } + else { + gc_page_profile_write_live_obj(s, v, page_profile_enabled); + } + v = (jl_taggedvalue_t*)((char*)v + osize); + } + v = v0; + } + // sweep the page while ((char*)v <= lim) { int bits = v->bits.gc; // if an object is past `lim_newpages` then we can guarantee it's garbage @@ -1526,6 +1548,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag push_lf_back(&global_page_pool_lazily_freed, pg); } } + gc_page_profile_write_to_file(s); gc_update_page_fragmentation_data(pg); gc_time_count_page(freedall, pg_skpd); jl_ptls_t ptls = gc_all_tls_states[pg->thread_n]; @@ -1534,7 +1557,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag } // the actual sweeping over all allocated pages in a memory pool -STATIC_INLINE void gc_sweep_pool_page(jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed, +STATIC_INLINE void gc_sweep_pool_page(gc_page_profiler_serializer_t *s, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed, jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT { int p_n = pg->pool_n; @@ -1542,7 +1565,7 @@ STATIC_INLINE void gc_sweep_pool_page(jl_gc_page_stack_t *allocd, jl_gc_page_sta jl_ptls_t ptls2 = gc_all_tls_states[t_n]; jl_gc_pool_t *p = &ptls2->heap.norm_pools[p_n]; int osize = pg->osize; - gc_sweep_page(p, allocd, lazily_freed, pg, osize); + gc_sweep_page(s, p, allocd, lazily_freed, pg, osize); } // sweep over all memory that is being used and not in a pool @@ -1579,11 +1602,20 @@ void gc_sweep_wake_all(void) uv_mutex_unlock(&gc_threads_lock); } +void gc_sweep_wait_for_all(void) +{ + jl_atomic_store(&gc_allocd_scratch, NULL); + while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) { + jl_cpu_pause(); + } +} + void gc_sweep_pool_parallel(void) { jl_atomic_fetch_add(&gc_n_threads_sweeping, 1); jl_gc_page_stack_t *allocd_scratch = jl_atomic_load(&gc_allocd_scratch); if (allocd_scratch != NULL) { + gc_page_profiler_serializer_t serializer = gc_page_serializer_create(); while (1) { int found_pg = 0; for (int t_i = 0; t_i < gc_n_threads; t_i++) { @@ -1596,25 +1628,18 @@ void gc_sweep_pool_parallel(void) if (pg == NULL) { continue; } - gc_sweep_pool_page(allocd, &ptls2->page_metadata_buffered, pg); + gc_sweep_pool_page(&serializer, allocd, &ptls2->page_metadata_buffered, pg); found_pg = 1; } if (!found_pg) { break; } } + gc_page_serializer_destroy(&serializer); } jl_atomic_fetch_add(&gc_n_threads_sweeping, -1); } -void gc_sweep_wait_for_all(void) -{ - jl_atomic_store(&gc_allocd_scratch, NULL); - while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) { - jl_cpu_pause(); - } -} - void gc_free_pages(void) { while (1) { @@ -3778,6 +3803,7 @@ void jl_gc_init(void) { JL_MUTEX_INIT(&heapsnapshot_lock, "heapsnapshot_lock"); JL_MUTEX_INIT(&finalizers_lock, "finalizers_lock"); + uv_mutex_init(&page_profile_lock); uv_mutex_init(&gc_cache_lock); uv_mutex_init(&gc_perm_lock); uv_mutex_init(&gc_threads_lock); diff --git a/stdlib/Profile/src/Profile.jl b/stdlib/Profile/src/Profile.jl index 690d0465b3ec03..24d2512dcbedfd 100644 --- a/stdlib/Profile/src/Profile.jl +++ b/stdlib/Profile/src/Profile.jl @@ -1260,6 +1260,21 @@ function take_heap_snapshot(all_one::Bool=false; dir::Union{Nothing,S}=nothing) return take_heap_snapshot(fpath, all_one) end +""" + Profile.take_page_profile(io::IOStream) + Profile.take_page_profile(filepath::String) + +Write a JSON snapshot of the pages from Julia's pool allocator, printing for every pool allocated object, whether it's garbage, or its type. +""" +function take_page_profile(io::IOStream) + Base.@_lock_ios(io, ccall(:jl_gc_take_page_profile, Cvoid, (Ptr{Cvoid},), io.handle)) +end +function take_page_profile(filepath::String) + open(filepath, "w") do io + take_page_profile(io) + end + return filepath +end include("Allocs.jl") include("precompile.jl") diff --git a/stdlib/Profile/test/runtests.jl b/stdlib/Profile/test/runtests.jl index 2d6df81b1015d7..f4d64c791956fd 100644 --- a/stdlib/Profile/test/runtests.jl +++ b/stdlib/Profile/test/runtests.jl @@ -294,4 +294,14 @@ end rm(tmpdir, force = true, recursive = true) end +@testset "PageProfile" begin + fname = "$(getpid())_$(time_ns())" + fpath = joinpath(tempdir(), fname) + Profile.take_page_profile(fpath) + open(fpath) do fs + @test readline(fs) != "" + end + rm(fpath) +end + include("allocs.jl")