Skip to content

Commit

Permalink
page profile
Browse files Browse the repository at this point in the history
  • Loading branch information
d-netto committed Dec 21, 2023
1 parent c1e1d5c commit 6393ca1
Show file tree
Hide file tree
Showing 5 changed files with 273 additions and 13 deletions.
2 changes: 1 addition & 1 deletion src/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ SRCS := \
jltypes gf typemap smallintset ast builtins module interpreter symbol \
dlload sys init task array genericmemory staticdata toplevel jl_uv datatype \
simplevector runtime_intrinsics precompile jloptions mtarraylist \
threading scheduler stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler method \
threading scheduler stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler gc-page-profiler method \
jlapi signal-handling safepoint timing subtype rtutils gc-heap-snapshot \
crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall

Expand Down
171 changes: 171 additions & 0 deletions src/gc-page-profiler.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
// This file is a part of Julia. License is MIT: https://julialang.org/license

#include "gc-page-profiler.h"

#ifdef __cplusplus
extern "C" {
#endif

// whether page profiling is enabled
int page_profile_enabled;
// number of pages written
size_t page_profile_pages_written;
// stream to write page profile to
ios_t *page_profile_stream;
// mutex for page profile
uv_mutex_t page_profile_lock;

gc_page_profiler_serializer_t gc_page_serializer_create(void) JL_NOTSAFEPOINT
{
gc_page_profiler_serializer_t serializer;
if (__unlikely(page_profile_enabled)) {
arraylist_new(&serializer.typestrs, GC_PAGE_SZ);
}
else {
serializer.typestrs.len = 0;
}
return serializer;
}

void gc_page_serializer_init(gc_page_profiler_serializer_t *serializer,
jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT
{
if (__unlikely(page_profile_enabled)) {
serializer->typestrs.len = 0;
serializer->data = (char *)pg->data;
serializer->osize = pg->osize;
}
}

void gc_page_serializer_destroy(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT
{
if (__unlikely(page_profile_enabled)) {
arraylist_free(&serializer->typestrs);
}
}

void gc_page_serializer_write(gc_page_profiler_serializer_t *serializer,
const char *str) JL_NOTSAFEPOINT
{
if (__unlikely(page_profile_enabled)) {
arraylist_push(&serializer->typestrs, (void *)str);
}
}

void gc_enable_page_profile(void) JL_NOTSAFEPOINT
{
page_profile_enabled = 1;
}

void gc_disable_page_profile(void) JL_NOTSAFEPOINT
{
page_profile_enabled = 0;
}

int gc_page_profile_is_enabled(void) JL_NOTSAFEPOINT
{
return page_profile_enabled;
}

void gc_page_profile_write_preamble(gc_page_profiler_serializer_t *serializer)
JL_NOTSAFEPOINT
{
if (__unlikely(page_profile_enabled)) {
char str[GC_TYPE_STR_MAXLEN];
snprintf(str, GC_TYPE_STR_MAXLEN,
"{\"address\": \"%p\",\"object_size\": %d,\"objects\": [",
serializer->data, serializer->osize);
ios_write(page_profile_stream, str, strlen(str));
}
}

void gc_page_profile_write_epilogue(gc_page_profiler_serializer_t *serializer)
JL_NOTSAFEPOINT
{
if (__unlikely(page_profile_enabled)) {
char str[GC_TYPE_STR_MAXLEN];
snprintf(str, GC_TYPE_STR_MAXLEN, "]}");
ios_write(page_profile_stream, str, strlen(str));
}
}

void gc_page_profile_write_comma(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT
{
if (__unlikely(page_profile_enabled)) {
// write comma if not first page
if (page_profile_pages_written > 0) {
char str[GC_TYPE_STR_MAXLEN];
snprintf(str, GC_TYPE_STR_MAXLEN, ",");
ios_write(page_profile_stream, str, strlen(str));
}
}
}

void gc_page_profile_write_to_file(gc_page_profiler_serializer_t *serializer)
JL_NOTSAFEPOINT
{
if (__unlikely(page_profile_enabled)) {
// write to file
uv_mutex_lock(&page_profile_lock);
gc_page_profile_write_comma(serializer);
gc_page_profile_write_preamble(serializer);
char str[GC_TYPE_STR_MAXLEN];
for (size_t i = 0; i < serializer->typestrs.len; i++) {
const char *name = (const char *)serializer->typestrs.items[i];
if (name == GC_SERIALIZER_EMPTY) {
snprintf(str, GC_TYPE_STR_MAXLEN, "\"empty\",");
}
else if (name == GC_SERIALIZER_GARBAGE) {
snprintf(str, GC_TYPE_STR_MAXLEN, "\"garbage\",");
}
else {
snprintf(str, GC_TYPE_STR_MAXLEN, "\"%s\",", name);
}
// remove trailing comma for last element
if (i == serializer->typestrs.len - 1) {
str[strlen(str) - 1] = '\0';
}
ios_write(page_profile_stream, str, strlen(str));
}
gc_page_profile_write_epilogue(serializer);
page_profile_pages_written++;
uv_mutex_unlock(&page_profile_lock);
}
}

void gc_page_profile_write_json_preamble(ios_t *stream) JL_NOTSAFEPOINT
{
if (__unlikely(page_profile_enabled)) {
uv_mutex_lock(&page_profile_lock);
char str[GC_TYPE_STR_MAXLEN];
snprintf(str, GC_TYPE_STR_MAXLEN, "{\"pages\": [");
ios_write(stream, str, strlen(str));
uv_mutex_unlock(&page_profile_lock);
}
}

void gc_page_profile_write_json_epilogue(ios_t *stream) JL_NOTSAFEPOINT
{
if (__unlikely(page_profile_enabled)) {
uv_mutex_lock(&page_profile_lock);
char str[GC_TYPE_STR_MAXLEN];
snprintf(str, GC_TYPE_STR_MAXLEN, "]}");
ios_write(stream, str, strlen(str));
uv_mutex_unlock(&page_profile_lock);
}
}

JL_DLLEXPORT void jl_gc_take_page_profile(ios_t *stream)
{
gc_enable_page_profile();
page_profile_pages_written = 0;
page_profile_stream = stream;
gc_page_profile_write_json_preamble(stream);
jl_gc_collect(JL_GC_FULL);
gc_page_profile_write_json_epilogue(stream);
gc_disable_page_profile();
}

#ifdef __cplusplus
}
#endif
63 changes: 63 additions & 0 deletions src/gc-page-profiler.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// This file is a part of Julia. License is MIT: https://julialang.org/license

#ifndef GC_PAGE_PROFILER_H
#define GC_PAGE_PROFILER_H

#include "gc.h"

#ifdef __cplusplus
extern "C" {
#endif

#define GC_TYPE_STR_MAXLEN (512)

typedef struct {
arraylist_t typestrs;
char *data;
int osize;
} gc_page_profiler_serializer_t;

// mutex for page profile
extern uv_mutex_t page_profile_lock;

// Serializer functions
gc_page_profiler_serializer_t gc_page_serializer_create(void) JL_NOTSAFEPOINT;
void gc_page_serializer_init(gc_page_profiler_serializer_t *serializer, jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT;
void gc_page_serializer_destroy(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT;
void gc_page_serializer_write(gc_page_profiler_serializer_t *serializer, const char *str) JL_NOTSAFEPOINT;
// Page profile functions
#define GC_SERIALIZER_EMPTY ((const char *)0x1)
#define GC_SERIALIZER_GARBAGE ((const char *)0x2)
STATIC_INLINE void gc_page_profile_write_empty_page(gc_page_profiler_serializer_t *serializer,
int enabled) JL_NOTSAFEPOINT
{
if (__unlikely(enabled)) {
gc_page_serializer_write(serializer, GC_SERIALIZER_EMPTY);
}
}
STATIC_INLINE void gc_page_profile_write_garbage(gc_page_profiler_serializer_t *serializer,
int enabled) JL_NOTSAFEPOINT
{
if (__unlikely(enabled)) {
gc_page_serializer_write(serializer, GC_SERIALIZER_GARBAGE);
}
}
STATIC_INLINE void gc_page_profile_write_live_obj(gc_page_profiler_serializer_t *serializer,
jl_taggedvalue_t *v,
int enabled) JL_NOTSAFEPOINT
{
if (__unlikely(enabled)) {
const char *name = jl_typeof_str(jl_valueof(v));
gc_page_serializer_write(serializer, name);
}
}
void gc_enable_page_profile(void) JL_NOTSAFEPOINT;
void gc_disable_page_profile(void) JL_NOTSAFEPOINT;
int gc_page_profile_is_enabled(void) JL_NOTSAFEPOINT;
void gc_page_profile_write_to_file(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT;

#ifdef __cplusplus
}
#endif

#endif // GC_PAGE_PROFILER_H
36 changes: 24 additions & 12 deletions src/gc.c
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// This file is a part of Julia. License is MIT: https://julialang.org/license

#include "gc.h"
#include "gc-page-profiler.h"
#include "julia.h"
#include "julia_gcext.h"
#include "julia_assert.h"
Expand Down Expand Up @@ -1426,7 +1427,7 @@ STATIC_INLINE void gc_dump_page_utilization_data(void) JL_NOTSAFEPOINT
int64_t buffered_pages = 0;

// Returns pointer to terminal pointer of list rooted at *pfl.
static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered,
static void gc_sweep_page(gc_page_profiler_serializer_t *s, jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered,
jl_gc_pagemeta_t *pg, int osize) JL_NOTSAFEPOINT
{
char *data = pg->data;
Expand All @@ -1438,6 +1439,9 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
}
size_t old_nfree = pg->nfree;
size_t nfree;
gc_page_serializer_init(s, pg);
// avoid loading a global variable in the hot path
int enabled = gc_page_profile_is_enabled();

int re_use_page = 1;
int keep_as_local_buffer = 0;
Expand All @@ -1457,6 +1461,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
}
#endif
nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / osize;
gc_page_profile_write_empty_page(s, enabled);
goto done;
}
// For quick sweep, we might be able to skip the page if the page doesn't
Expand All @@ -1466,6 +1471,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
if (!prev_sweep_full || pg->prev_nold == pg->nold) {
freedall = 0;
nfree = pg->nfree;
gc_page_profile_write_empty_page(s, enabled);
goto done;
}
}
Expand All @@ -1483,12 +1489,14 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
int bits = v->bits.gc;
// if an object is past `lim_newpages` then we can guarantee it's garbage
if (!gc_marked(bits) || (char*)v >= lim_newpages) {
gc_page_profile_write_garbage(s, enabled);
*pfl = v;
pfl = &v->next;
pfl_begin = (pfl_begin != NULL) ? pfl_begin : pfl;
pg_nfree++;
}
else { // marked young or old
gc_page_profile_write_live_obj(s, v, enabled);
if (current_sweep_full || bits == GC_MARKED) { // old enough
bits = v->bits.gc = GC_OLD; // promote
}
Expand Down Expand Up @@ -1532,6 +1540,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
push_lf_back(&global_page_pool_lazily_freed, pg);
}
}
gc_page_profile_write_to_file(s);
gc_update_page_fragmentation_data(pg);
gc_time_count_page(freedall, pg_skpd);
jl_ptls_t ptls = gc_all_tls_states[pg->thread_n];
Expand All @@ -1540,15 +1549,15 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
}

// the actual sweeping over all allocated pages in a memory pool
STATIC_INLINE void gc_sweep_pool_page(jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed,
STATIC_INLINE void gc_sweep_pool_page(gc_page_profiler_serializer_t *s, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed,
jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT
{
int p_n = pg->pool_n;
int t_n = pg->thread_n;
jl_ptls_t ptls2 = gc_all_tls_states[t_n];
jl_gc_pool_t *p = &ptls2->heap.norm_pools[p_n];
int osize = pg->osize;
gc_sweep_page(p, allocd, lazily_freed, pg, osize);
gc_sweep_page(s, p, allocd, lazily_freed, pg, osize);
}

// sweep over all memory that is being used and not in a pool
Expand Down Expand Up @@ -1585,11 +1594,20 @@ void gc_sweep_wake_all(void)
uv_mutex_unlock(&gc_threads_lock);
}

void gc_sweep_wait_for_all(void)
{
jl_atomic_store(&gc_allocd_scratch, NULL);
while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) {
jl_cpu_pause();
}
}

void gc_sweep_pool_parallel(void)
{
jl_atomic_fetch_add(&gc_n_threads_sweeping, 1);
jl_gc_page_stack_t *allocd_scratch = jl_atomic_load(&gc_allocd_scratch);
if (allocd_scratch != NULL) {
gc_page_profiler_serializer_t serializer = gc_page_serializer_create();
while (1) {
int found_pg = 0;
for (int t_i = 0; t_i < gc_n_threads; t_i++) {
Expand All @@ -1602,25 +1620,18 @@ void gc_sweep_pool_parallel(void)
if (pg == NULL) {
continue;
}
gc_sweep_pool_page(allocd, &ptls2->page_metadata_buffered, pg);
gc_sweep_pool_page(&serializer, allocd, &ptls2->page_metadata_buffered, pg);
found_pg = 1;
}
if (!found_pg) {
break;
}
}
gc_page_serializer_destroy(&serializer);
}
jl_atomic_fetch_add(&gc_n_threads_sweeping, -1);
}

void gc_sweep_wait_for_all(void)
{
jl_atomic_store(&gc_allocd_scratch, NULL);
while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) {
jl_cpu_pause();
}
}

void gc_free_pages(void)
{
while (1) {
Expand Down Expand Up @@ -3862,6 +3873,7 @@ void jl_gc_init(void)
{
JL_MUTEX_INIT(&heapsnapshot_lock, "heapsnapshot_lock");
JL_MUTEX_INIT(&finalizers_lock, "finalizers_lock");
uv_mutex_init(&page_profile_lock);
uv_mutex_init(&gc_cache_lock);
uv_mutex_init(&gc_perm_lock);
uv_mutex_init(&gc_threads_lock);
Expand Down
Loading

0 comments on commit 6393ca1

Please sign in to comment.