From 2deb6cff84c8063df8fab764a1e01306f9449391 Mon Sep 17 00:00:00 2001 From: d-netto Date: Sun, 17 Dec 2023 14:10:38 -0300 Subject: [PATCH] page profile --- src/Makefile | 2 +- src/gc-page-profiler.c | 171 ++++++++++++++++++++++++++++++++ src/gc-page-profiler.h | 63 ++++++++++++ src/gc.c | 53 +++++++--- stdlib/Profile/src/Profile.jl | 16 +++ stdlib/Profile/test/runtests.jl | 10 ++ 6 files changed, 300 insertions(+), 15 deletions(-) create mode 100644 src/gc-page-profiler.c create mode 100644 src/gc-page-profiler.h diff --git a/src/Makefile b/src/Makefile index 747090a8dcd831..c78df75bdceb50 100644 --- a/src/Makefile +++ b/src/Makefile @@ -44,7 +44,7 @@ SRCS := \ jltypes gf typemap smallintset ast builtins module interpreter symbol \ dlload sys init task array genericmemory staticdata toplevel jl_uv datatype \ simplevector runtime_intrinsics precompile jloptions mtarraylist \ - threading scheduler stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler method \ + threading scheduler stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler gc-page-profiler method \ jlapi signal-handling safepoint timing subtype rtutils gc-heap-snapshot \ crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall diff --git a/src/gc-page-profiler.c b/src/gc-page-profiler.c new file mode 100644 index 00000000000000..94920c408a45d8 --- /dev/null +++ b/src/gc-page-profiler.c @@ -0,0 +1,171 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include "gc-page-profiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// whether page profiling is enabled +int page_profile_enabled; +// number of pages written +size_t page_profile_pages_written; +// stream to write page profile to +ios_t *page_profile_stream; +// mutex for page profile +uv_mutex_t page_profile_lock; + +gc_page_profiler_serializer_t gc_page_serializer_create(void) JL_NOTSAFEPOINT +{ + gc_page_profiler_serializer_t serializer; + if (__unlikely(page_profile_enabled)) { + arraylist_new(&serializer.typestrs, GC_PAGE_SZ); + } + else { + serializer.typestrs.len = 0; + } + return serializer; +} + +void gc_page_serializer_init(gc_page_profiler_serializer_t *serializer, + jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + serializer->typestrs.len = 0; + serializer->data = (char *)pg->data; + serializer->osize = pg->osize; + } +} + +void gc_page_serializer_destroy(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + arraylist_free(&serializer->typestrs); + } +} + +void gc_page_serializer_write(gc_page_profiler_serializer_t *serializer, + const char *str) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + arraylist_push(&serializer->typestrs, (void *)str); + } +} + +void gc_enable_page_profile(void) JL_NOTSAFEPOINT +{ + page_profile_enabled = 1; +} + +void gc_disable_page_profile(void) JL_NOTSAFEPOINT +{ + page_profile_enabled = 0; +} + +int gc_page_profile_is_enabled(void) JL_NOTSAFEPOINT +{ + return page_profile_enabled; +} + +void gc_page_profile_write_preamble(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + char str[GC_TYPE_STR_MAXLEN]; + snprintf(str, GC_TYPE_STR_MAXLEN, + "{\"address\": \"%p\",\"object_size\": %d,\"objects\": [", + serializer->data, serializer->osize); + ios_write(page_profile_stream, str, strlen(str)); + } +} + +void gc_page_profile_write_epilogue(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + char str[GC_TYPE_STR_MAXLEN]; + snprintf(str, GC_TYPE_STR_MAXLEN, "]}"); + ios_write(page_profile_stream, str, strlen(str)); + } +} + +void gc_page_profile_write_comma(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + // write comma if not first page + if (page_profile_pages_written > 0) { + char str[GC_TYPE_STR_MAXLEN]; + snprintf(str, GC_TYPE_STR_MAXLEN, ","); + ios_write(page_profile_stream, str, strlen(str)); + } + } +} + +void gc_page_profile_write_to_file(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + // write to file + uv_mutex_lock(&page_profile_lock); + gc_page_profile_write_comma(serializer); + gc_page_profile_write_preamble(serializer); + char str[GC_TYPE_STR_MAXLEN]; + for (size_t i = 0; i < serializer->typestrs.len; i++) { + const char *name = (const char *)serializer->typestrs.items[i]; + if (name == GC_SERIALIZER_EMPTY) { + snprintf(str, GC_TYPE_STR_MAXLEN, "\"empty\","); + } + else if (name == GC_SERIALIZER_GARBAGE) { + snprintf(str, GC_TYPE_STR_MAXLEN, "\"garbage\","); + } + else { + snprintf(str, GC_TYPE_STR_MAXLEN, "\"%s\",", name); + } + // remove trailing comma for last element + if (i == serializer->typestrs.len - 1) { + str[strlen(str) - 1] = '\0'; + } + ios_write(page_profile_stream, str, strlen(str)); + } + gc_page_profile_write_epilogue(serializer); + page_profile_pages_written++; + uv_mutex_unlock(&page_profile_lock); + } +} + +void gc_page_profile_write_json_preamble(ios_t *stream) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + uv_mutex_lock(&page_profile_lock); + char str[GC_TYPE_STR_MAXLEN]; + snprintf(str, GC_TYPE_STR_MAXLEN, "{\"pages\": ["); + ios_write(stream, str, strlen(str)); + uv_mutex_unlock(&page_profile_lock); + } +} + +void gc_page_profile_write_json_epilogue(ios_t *stream) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + uv_mutex_lock(&page_profile_lock); + char str[GC_TYPE_STR_MAXLEN]; + snprintf(str, GC_TYPE_STR_MAXLEN, "]}"); + ios_write(stream, str, strlen(str)); + uv_mutex_unlock(&page_profile_lock); + } +} + +JL_DLLEXPORT void jl_gc_take_page_profile(ios_t *stream) +{ + gc_enable_page_profile(); + page_profile_pages_written = 0; + page_profile_stream = stream; + gc_page_profile_write_json_preamble(stream); + jl_gc_collect(JL_GC_FULL); + gc_page_profile_write_json_epilogue(stream); + gc_disable_page_profile(); +} + +#ifdef __cplusplus +} +#endif diff --git a/src/gc-page-profiler.h b/src/gc-page-profiler.h new file mode 100644 index 00000000000000..b103e23905ba52 --- /dev/null +++ b/src/gc-page-profiler.h @@ -0,0 +1,63 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#ifndef GC_PAGE_PROFILER_H +#define GC_PAGE_PROFILER_H + +#include "gc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define GC_TYPE_STR_MAXLEN (512) + +typedef struct { + arraylist_t typestrs; + char *data; + int osize; +} gc_page_profiler_serializer_t; + +// mutex for page profile +extern uv_mutex_t page_profile_lock; + +// Serializer functions +gc_page_profiler_serializer_t gc_page_serializer_create(void) JL_NOTSAFEPOINT; +void gc_page_serializer_init(gc_page_profiler_serializer_t *serializer, jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT; +void gc_page_serializer_destroy(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; +void gc_page_serializer_write(gc_page_profiler_serializer_t *serializer, const char *str) JL_NOTSAFEPOINT; +// Page profile functions +#define GC_SERIALIZER_EMPTY ((const char *)0x1) +#define GC_SERIALIZER_GARBAGE ((const char *)0x2) +STATIC_INLINE void gc_page_profile_write_empty_page(gc_page_profiler_serializer_t *serializer, + int enabled) JL_NOTSAFEPOINT +{ + if (__unlikely(enabled)) { + gc_page_serializer_write(serializer, GC_SERIALIZER_EMPTY); + } +} +STATIC_INLINE void gc_page_profile_write_garbage(gc_page_profiler_serializer_t *serializer, + int enabled) JL_NOTSAFEPOINT +{ + if (__unlikely(enabled)) { + gc_page_serializer_write(serializer, GC_SERIALIZER_GARBAGE); + } +} +STATIC_INLINE void gc_page_profile_write_live_obj(gc_page_profiler_serializer_t *serializer, + jl_taggedvalue_t *v, + int enabled) JL_NOTSAFEPOINT +{ + if (__unlikely(enabled)) { + const char *name = jl_typeof_str(jl_valueof(v)); + gc_page_serializer_write(serializer, name); + } +} +void gc_enable_page_profile(void) JL_NOTSAFEPOINT; +void gc_disable_page_profile(void) JL_NOTSAFEPOINT; +int gc_page_profile_is_enabled(void) JL_NOTSAFEPOINT; +void gc_page_profile_write_to_file(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; + +#ifdef __cplusplus +} +#endif + +#endif // GC_PAGE_PROFILER_H diff --git a/src/gc.c b/src/gc.c index d703fb9d262182..d4c068dc7eab80 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1,6 +1,7 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "gc.h" +#include "gc-page-profiler.h" #include "julia.h" #include "julia_gcext.h" #include "julia_assert.h" @@ -1426,7 +1427,7 @@ STATIC_INLINE void gc_dump_page_utilization_data(void) JL_NOTSAFEPOINT int64_t buffered_pages = 0; // Returns pointer to terminal pointer of list rooted at *pfl. -static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered, +static void gc_sweep_page(gc_page_profiler_serializer_t *s, jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered, jl_gc_pagemeta_t *pg, int osize) JL_NOTSAFEPOINT { char *data = pg->data; @@ -1438,6 +1439,9 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag } size_t old_nfree = pg->nfree; size_t nfree; + // avoid loading a global variable in the hot path + int enabled = gc_page_profile_is_enabled(); + gc_page_serializer_init(s, pg); int re_use_page = 1; int keep_as_local_buffer = 0; @@ -1457,6 +1461,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag } #endif nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / osize; + gc_page_profile_write_empty_page(s, enabled); goto done; } // For quick sweep, we might be able to skip the page if the page doesn't @@ -1466,12 +1471,13 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag if (!prev_sweep_full || pg->prev_nold == pg->nold) { freedall = 0; nfree = pg->nfree; + gc_page_profile_write_empty_page(s, enabled); goto done; } } pg_skpd = 0; - { // scope to avoid clang goto errors + { // scope to avoid clang goto errors int has_marked = 0; int has_young = 0; int16_t prev_nold = 0; @@ -1479,9 +1485,24 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag jl_taggedvalue_t *fl = NULL; jl_taggedvalue_t **pfl = &fl; jl_taggedvalue_t **pfl_begin = NULL; + // collect page profile + if (enabled) { + while ((char*)v <= lim) { + int bits = v->bits.gc; + // if an object is past `lim_newpages` then we can guarantee it's garbage + if (!gc_marked(bits) || (char*)v >= lim_newpages) { + gc_page_profile_write_garbage(s, enabled); + } + else { + gc_page_profile_write_live_obj(s, v, enabled); + } + v = (jl_taggedvalue_t*)((char*)v + osize); + } + } + v = (jl_taggedvalue_t*)(data + GC_PAGE_OFFSET); + // sweep the page while ((char*)v <= lim) { int bits = v->bits.gc; - // if an object is past `lim_newpages` then we can guarantee it's garbage if (!gc_marked(bits) || (char*)v >= lim_newpages) { *pfl = v; pfl = &v->next; @@ -1532,6 +1553,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag push_lf_back(&global_page_pool_lazily_freed, pg); } } + gc_page_profile_write_to_file(s); gc_update_page_fragmentation_data(pg); gc_time_count_page(freedall, pg_skpd); jl_ptls_t ptls = gc_all_tls_states[pg->thread_n]; @@ -1540,7 +1562,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag } // the actual sweeping over all allocated pages in a memory pool -STATIC_INLINE void gc_sweep_pool_page(jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed, +STATIC_INLINE void gc_sweep_pool_page(gc_page_profiler_serializer_t *s, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed, jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT { int p_n = pg->pool_n; @@ -1548,7 +1570,7 @@ STATIC_INLINE void gc_sweep_pool_page(jl_gc_page_stack_t *allocd, jl_gc_page_sta jl_ptls_t ptls2 = gc_all_tls_states[t_n]; jl_gc_pool_t *p = &ptls2->heap.norm_pools[p_n]; int osize = pg->osize; - gc_sweep_page(p, allocd, lazily_freed, pg, osize); + gc_sweep_page(s, p, allocd, lazily_freed, pg, osize); } // sweep over all memory that is being used and not in a pool @@ -1585,11 +1607,20 @@ void gc_sweep_wake_all(void) uv_mutex_unlock(&gc_threads_lock); } +void gc_sweep_wait_for_all(void) +{ + jl_atomic_store(&gc_allocd_scratch, NULL); + while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) { + jl_cpu_pause(); + } +} + void gc_sweep_pool_parallel(void) { jl_atomic_fetch_add(&gc_n_threads_sweeping, 1); jl_gc_page_stack_t *allocd_scratch = jl_atomic_load(&gc_allocd_scratch); if (allocd_scratch != NULL) { + gc_page_profiler_serializer_t serializer = gc_page_serializer_create(); while (1) { int found_pg = 0; for (int t_i = 0; t_i < gc_n_threads; t_i++) { @@ -1602,25 +1633,18 @@ void gc_sweep_pool_parallel(void) if (pg == NULL) { continue; } - gc_sweep_pool_page(allocd, &ptls2->page_metadata_buffered, pg); + gc_sweep_pool_page(&serializer, allocd, &ptls2->page_metadata_buffered, pg); found_pg = 1; } if (!found_pg) { break; } } + gc_page_serializer_destroy(&serializer); } jl_atomic_fetch_add(&gc_n_threads_sweeping, -1); } -void gc_sweep_wait_for_all(void) -{ - jl_atomic_store(&gc_allocd_scratch, NULL); - while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) { - jl_cpu_pause(); - } -} - void gc_free_pages(void) { while (1) { @@ -3862,6 +3886,7 @@ void jl_gc_init(void) { JL_MUTEX_INIT(&heapsnapshot_lock, "heapsnapshot_lock"); JL_MUTEX_INIT(&finalizers_lock, "finalizers_lock"); + uv_mutex_init(&page_profile_lock); uv_mutex_init(&gc_cache_lock); uv_mutex_init(&gc_perm_lock); uv_mutex_init(&gc_threads_lock); diff --git a/stdlib/Profile/src/Profile.jl b/stdlib/Profile/src/Profile.jl index 206d2957a91e56..cc29a4503d8105 100644 --- a/stdlib/Profile/src/Profile.jl +++ b/stdlib/Profile/src/Profile.jl @@ -16,6 +16,7 @@ public @profile, callers, init, take_heap_snapshot, + take_page_profile, clear_malloc_data, Allocs @@ -1267,6 +1268,21 @@ function take_heap_snapshot(all_one::Bool=false; dir::Union{Nothing,S}=nothing) return take_heap_snapshot(fpath, all_one) end +""" + Profile.take_page_profile(io::IOStream) + Profile.take_page_profile(filepath::String) + +Write a JSON snapshot of the pages from Julia's pool allocator, printing for every pool allocated object, whether it's garbage, or its type. +""" +function take_page_profile(io::IOStream) + Base.@_lock_ios(io, ccall(:jl_gc_take_page_profile, Cvoid, (Ptr{Cvoid},), io.handle)) +end +function take_page_profile(filepath::String) + open(filepath, "w") do io + take_page_profile(io) + end + return filepath +end include("Allocs.jl") include("precompile.jl") diff --git a/stdlib/Profile/test/runtests.jl b/stdlib/Profile/test/runtests.jl index ad7c12272d051b..d7358dc20e853c 100644 --- a/stdlib/Profile/test/runtests.jl +++ b/stdlib/Profile/test/runtests.jl @@ -289,4 +289,14 @@ end rm(tmpdir, force = true, recursive = true) end +@testset "PageProfile" begin + fname = "$(getpid())_$(time_ns())" + fpath = joinpath(tempdir(), fname) + Profile.take_page_profile(fpath) + open(fpath) do fs + @test readline(fs) != "" + end + rm(fpath) +end + include("allocs.jl")