From 4613cf505bfd5ac62b682c9072291ad6d95d5f64 Mon Sep 17 00:00:00 2001 From: d-netto Date: Sun, 17 Dec 2023 14:10:38 -0300 Subject: [PATCH] page profile --- src/Makefile | 2 +- src/gc-page-profiler.c | 211 ++++++++++++++++++++++++++++++++++ src/gc-page-profiler.h | 44 +++++++ src/gc.c | 34 ++++-- stdlib/Profile/src/Profile.jl | 13 +++ 5 files changed, 291 insertions(+), 13 deletions(-) create mode 100644 src/gc-page-profiler.c create mode 100644 src/gc-page-profiler.h diff --git a/src/Makefile b/src/Makefile index 42ff9aa3a9b361..2a66e3a0dccac5 100644 --- a/src/Makefile +++ b/src/Makefile @@ -44,7 +44,7 @@ SRCS := \ jltypes gf typemap smallintset ast builtins module interpreter symbol \ dlload sys init task array staticdata toplevel jl_uv datatype \ simplevector runtime_intrinsics precompile jloptions mtarraylist \ - threading partr stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler method \ + threading partr stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler gc-page-profiler method \ jlapi signal-handling safepoint timing subtype rtutils gc-heap-snapshot \ crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall diff --git a/src/gc-page-profiler.c b/src/gc-page-profiler.c new file mode 100644 index 00000000000000..da0f277e2b65f0 --- /dev/null +++ b/src/gc-page-profiler.c @@ -0,0 +1,211 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include "gc-page-profiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// whether page profiling is enabled +int page_profile_enabled; +// number of pages written +size_t page_profile_pages_written; +// stream to write page profile to +ios_t *page_profile_stream; +// mutex for page profile +uv_mutex_t page_profile_lock; + +#define GC_SERIALIZER_EMPTY ((const char *)0x1) +#define GC_SERIALIZER_GARBAGE ((const char *)0x2) + +gc_page_profiler_serializer_t gc_page_serializer_create(void) JL_NOTSAFEPOINT +{ + gc_page_profiler_serializer_t serializer; + serializer.length = 0; + serializer.capacity = GC_PAGE_SZ; + if (__unlikely(page_profile_enabled)) { + serializer.buffer = (char const **)calloc_s(serializer.capacity); + } + else { + serializer.buffer = NULL; + } + return serializer; +} + +void gc_page_serializer_init(gc_page_profiler_serializer_t *serializer, + jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + memset(serializer->buffer, 0, serializer->capacity); + serializer->length = 0; + serializer->data = (char *)pg->data; + serializer->osize = pg->osize; + } +} + +void gc_page_serializer_destroy(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT +{ + free(serializer->buffer); +} + +void gc_page_serializer_write(gc_page_profiler_serializer_t *serializer, + const char *str) JL_NOTSAFEPOINT +{ + serializer->buffer[serializer->length++] = str; +} + +void gc_enable_page_profile(void) JL_NOTSAFEPOINT +{ + page_profile_enabled = 1; +} + +void gc_disable_page_profile(void) JL_NOTSAFEPOINT +{ + page_profile_enabled = 0; +} + +void gc_page_profile_write_empty_page(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + gc_page_serializer_write(serializer, GC_SERIALIZER_EMPTY); + } +} + +void gc_page_profile_write_garbage(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + gc_page_serializer_write(serializer, GC_SERIALIZER_GARBAGE); + } +} + +void gc_page_profile_write_live_obj(gc_page_profiler_serializer_t *serializer, + jl_taggedvalue_t *v) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + const char *name = jl_typeof_str(jl_valueof(v)); + gc_page_serializer_write(serializer, name); + } +} + +void gc_page_profile_write_preamble(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + char str[GC_TYPE_STR_MAXLEN]; + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "{"); + ios_write(page_profile_stream, str, strlen(str)); + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "\"address\": \"%p\",", serializer->data); + ios_write(page_profile_stream, str, strlen(str)); + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "\"object_size\": %d,", serializer->osize); + ios_write(page_profile_stream, str, strlen(str)); + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "\"objects\": ["); + ios_write(page_profile_stream, str, strlen(str)); + } +} + +void gc_page_profile_write_epilogue(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + char str[GC_TYPE_STR_MAXLEN]; + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "]"); + ios_write(page_profile_stream, str, strlen(str)); + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "}"); + ios_write(page_profile_stream, str, strlen(str)); + } +} + +void gc_page_profile_write_comma(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + if (page_profile_pages_written > 0) { + char str[GC_TYPE_STR_MAXLEN]; + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, ","); + ios_write(page_profile_stream, str, strlen(str)); + } + } +} + +void gc_page_profile_write_to_file(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + // write to file + uv_mutex_lock(&page_profile_lock); + gc_page_profile_write_comma(serializer); + gc_page_profile_write_preamble(serializer); + char str[GC_TYPE_STR_MAXLEN]; + for (size_t i = 0; i < serializer->length; i++) { + memset(str, 0, GC_TYPE_STR_MAXLEN); + if (serializer->buffer[i] == GC_SERIALIZER_EMPTY) { + snprintf(str, GC_TYPE_STR_MAXLEN, "\"empty\","); + } + else if (serializer->buffer[i] == GC_SERIALIZER_GARBAGE) { + snprintf(str, GC_TYPE_STR_MAXLEN, "\"garbage\","); + } + else { + snprintf(str, GC_TYPE_STR_MAXLEN, "\"%s\",", serializer->buffer[i]); + } + if (i == serializer->length - 1) { + str[strlen(str) - 1] = '\0'; + } + ios_write(page_profile_stream, str, strlen(str)); + } + gc_page_profile_write_epilogue(serializer); + page_profile_pages_written++; + uv_mutex_unlock(&page_profile_lock); + } +} + +void gc_page_profile_write_json_preamble(ios_t *stream) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + uv_mutex_lock(&page_profile_lock); + char str[GC_TYPE_STR_MAXLEN]; + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "{"); + ios_write(stream, str, strlen(str)); + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "\"pages\": ["); + ios_write(stream, str, strlen(str)); + uv_mutex_unlock(&page_profile_lock); + } +} + +void gc_page_profile_write_json_epilogue(ios_t *stream) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + uv_mutex_lock(&page_profile_lock); + char str[GC_TYPE_STR_MAXLEN]; + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "]"); + ios_write(stream, str, strlen(str)); + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "}"); + ios_write(stream, str, strlen(str)); + uv_mutex_unlock(&page_profile_lock); + } +} + +JL_DLLEXPORT void jl_gc_take_page_profile(ios_t *stream) +{ + gc_enable_page_profile(); + page_profile_stream = stream; + gc_page_profile_write_json_preamble(stream); + jl_gc_collect(JL_GC_FULL); + gc_page_profile_write_json_epilogue(stream); + gc_disable_page_profile(); +} + +#ifdef __cplusplus +} +#endif diff --git a/src/gc-page-profiler.h b/src/gc-page-profiler.h new file mode 100644 index 00000000000000..0307f1670daa75 --- /dev/null +++ b/src/gc-page-profiler.h @@ -0,0 +1,44 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#ifndef GC_PAGE_PROFILER_H +#define GC_PAGE_PROFILER_H + +#include "gc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define GC_TYPE_STR_MAXLEN (512) + +typedef struct { + size_t length; + size_t capacity; + char const **buffer; + char *data; + int osize; +} gc_page_profiler_serializer_t; + +// mutex for page profile +extern uv_mutex_t page_profile_lock; + +// Serializer functions +gc_page_profiler_serializer_t gc_page_serializer_create(void) JL_NOTSAFEPOINT; +void gc_page_serializer_init(gc_page_profiler_serializer_t *serializer, jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT; +void gc_page_serializer_destroy(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; +void gc_page_serializer_write(gc_page_profiler_serializer_t *serializer, const char *str) JL_NOTSAFEPOINT; +// Page profile functions +void gc_page_profile_write_preamble(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; +void gc_page_profile_write_epilogue(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; +void gc_page_profile_write_empty_page(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; +void gc_page_profile_write_garbage(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; +void gc_page_profile_write_live_obj(gc_page_profiler_serializer_t *serializer, jl_taggedvalue_t *v) JL_NOTSAFEPOINT; +void gc_enable_page_profile(void) JL_NOTSAFEPOINT; +void gc_disable_page_profile(void) JL_NOTSAFEPOINT; +void gc_page_profile_write_to_file(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; + +#ifdef __cplusplus +} +#endif + +#endif // GC_PAGE_PROFILER_H diff --git a/src/gc.c b/src/gc.c index cb2bab9acd59ff..206e8e1f4ccd66 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1,6 +1,7 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "gc.h" +#include "gc-page-profiler.h" #include "julia_gcext.h" #include "julia_assert.h" #ifdef __GLIBC__ @@ -1509,7 +1510,7 @@ STATIC_INLINE void gc_dump_page_utilization_data(void) JL_NOTSAFEPOINT int64_t buffered_pages = 0; // Returns pointer to terminal pointer of list rooted at *pfl. -static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered, +static void gc_sweep_page(gc_page_profiler_serializer_t *s, jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered, jl_gc_pagemeta_t *pg, int osize) JL_NOTSAFEPOINT { char *data = pg->data; @@ -1521,6 +1522,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag } size_t old_nfree = pg->nfree; size_t nfree; + gc_page_serializer_init(s, pg); int re_use_page = 1; int keep_as_local_buffer = 0; @@ -1538,6 +1540,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag keep_as_local_buffer = 1; } nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / osize; + gc_page_profile_write_empty_page(s); goto done; } // For quick sweep, we might be able to skip the page if the page doesn't @@ -1547,6 +1550,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag if (!prev_sweep_full || pg->prev_nold == pg->nold) { freedall = 0; nfree = pg->nfree; + gc_page_profile_write_empty_page(s); goto done; } } @@ -1564,12 +1568,14 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag int bits = v->bits.gc; // if an object is past `lim_newpages` then we can guarantee it's garbage if (!gc_marked(bits) || (char*)v >= lim_newpages) { + gc_page_profile_write_garbage(s); *pfl = v; pfl = &v->next; pfl_begin = (pfl_begin != NULL) ? pfl_begin : pfl; pg_nfree++; } else { // marked young or old + gc_page_profile_write_live_obj(s, v); if (current_sweep_full || bits == GC_MARKED) { // old enough bits = v->bits.gc = GC_OLD; // promote } @@ -1612,6 +1618,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag push_lf_back(&global_page_pool_lazily_freed, pg); } } + gc_page_profile_write_to_file(s); gc_update_page_fragmentation_data(pg); gc_time_count_page(freedall, pg_skpd); jl_ptls_t ptls = gc_all_tls_states[pg->thread_n]; @@ -1620,7 +1627,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag } // the actual sweeping over all allocated pages in a memory pool -STATIC_INLINE void gc_sweep_pool_page(jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed, +STATIC_INLINE void gc_sweep_pool_page(gc_page_profiler_serializer_t *s, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed, jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT { int p_n = pg->pool_n; @@ -1628,7 +1635,7 @@ STATIC_INLINE void gc_sweep_pool_page(jl_gc_page_stack_t *allocd, jl_gc_page_sta jl_ptls_t ptls2 = gc_all_tls_states[t_n]; jl_gc_pool_t *p = &ptls2->heap.norm_pools[p_n]; int osize = pg->osize; - gc_sweep_page(p, allocd, lazily_freed, pg, osize); + gc_sweep_page(s, p, allocd, lazily_freed, pg, osize); } // sweep over all memory that is being used and not in a pool @@ -1665,11 +1672,20 @@ void gc_sweep_wake_all(void) uv_mutex_unlock(&gc_threads_lock); } +void gc_sweep_wait_for_all(void) +{ + jl_atomic_store(&gc_allocd_scratch, NULL); + while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) { + jl_cpu_pause(); + } +} + void gc_sweep_pool_parallel(void) { jl_atomic_fetch_add(&gc_n_threads_sweeping, 1); jl_gc_page_stack_t *allocd_scratch = jl_atomic_load(&gc_allocd_scratch); if (allocd_scratch != NULL) { + gc_page_profiler_serializer_t serializer = gc_page_serializer_create(); while (1) { int found_pg = 0; for (int t_i = 0; t_i < gc_n_threads; t_i++) { @@ -1682,25 +1698,18 @@ void gc_sweep_pool_parallel(void) if (pg == NULL) { continue; } - gc_sweep_pool_page(allocd, &ptls2->page_metadata_buffered, pg); + gc_sweep_pool_page(&serializer, allocd, &ptls2->page_metadata_buffered, pg); found_pg = 1; } if (!found_pg) { break; } } + gc_page_serializer_destroy(&serializer); } jl_atomic_fetch_add(&gc_n_threads_sweeping, -1); } -void gc_sweep_wait_for_all(void) -{ - jl_atomic_store(&gc_allocd_scratch, NULL); - while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) { - jl_cpu_pause(); - } -} - void gc_free_pages(void) { while (1) { @@ -3825,6 +3834,7 @@ void jl_gc_init(void) { JL_MUTEX_INIT(&heapsnapshot_lock); JL_MUTEX_INIT(&finalizers_lock); + uv_mutex_init(&page_profile_lock); uv_mutex_init(&gc_cache_lock); uv_mutex_init(&gc_perm_lock); uv_mutex_init(&gc_threads_lock); diff --git a/stdlib/Profile/src/Profile.jl b/stdlib/Profile/src/Profile.jl index 6dd2a12205b66d..2692039bda5389 100644 --- a/stdlib/Profile/src/Profile.jl +++ b/stdlib/Profile/src/Profile.jl @@ -1275,6 +1275,19 @@ function take_heap_snapshot(all_one::Bool=false) return take_heap_snapshot(f, all_one) end +""" + Profile.take_page_profile(io::IOStream) + Profile.take_page_profile(filepath::String) +""" +function take_page_profile(io::IOStream) + Base.@_lock_ios(io, ccall(:jl_gc_take_page_profile, Cvoid, (Ptr{Cvoid},), io.handle)) +end +function take_page_profile(filepath::String) + open(filepath, "w") do io + take_page_profile(io) + end + return filepath +end include("Allocs.jl")