page profile

src/Makefile

@@ -44,7 +44,7 @@ SRCS := \
 	jltypes gf typemap smallintset ast builtins module interpreter symbol \
 	dlload sys init task array genericmemory staticdata toplevel jl_uv datatype \
 	simplevector runtime_intrinsics precompile jloptions mtarraylist \
-	threading scheduler stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler method \
+	threading scheduler stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler gc-page-profiler method \
 	jlapi signal-handling safepoint timing subtype rtutils gc-heap-snapshot \
 	crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall
 

src/gc-page-profiler.c

@@ -0,0 +1,146 @@
+// This file is a part of Julia. License is MIT: https://julialang.org/license
+
+#include "gc-page-profiler.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// whether page profiling is enabled
+int page_profile_enabled;
+// stream to write page profile to
+ios_t *page_profile_stream;
+// mutex for page profile
+uv_mutex_t page_profile_lock;
+
+gc_page_profiler_serializer_t gc_page_serializer_create(void) JL_NOTSAFEPOINT
+{
+    gc_page_profiler_serializer_t serializer;
+    serializer.length = 0;
+    serializer.capacity = GC_PAGE_SZ * GC_TYPE_STR_MAXLEN;
+    if (__unlikely(page_profile_enabled)) {
+        serializer.buffer = (char*)calloc_s(serializer.capacity);
+    }
+    else {
+        serializer.buffer = NULL;
+    }
+    return serializer;
+}
+
+void gc_page_serializer_init(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT
+{
+    if (__unlikely(page_profile_enabled)) {
+        memset(serializer->buffer, 0, serializer->capacity);
+        serializer->length = 0;
+    }
+}
+
+void gc_page_serializer_destroy(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT
+{
+    free(serializer->buffer);
+}
+
+void gc_page_serializer_write(gc_page_profiler_serializer_t *serializer,
+                              const char *str) JL_NOTSAFEPOINT
+{
+    size_t len = strlen(str);
+    if (serializer->length + len + 1 > serializer->capacity) {
+        jl_safe_printf("gc_page_serializer_write: serializer overflow\n");
+        abort();
+    }
+    memcpy(serializer->buffer + serializer->length, str, len);
+    serializer->length += len;
+    serializer->buffer[serializer->length] = '\n';
+    serializer->length++;
+}
+
+void gc_enable_page_profile(void) JL_NOTSAFEPOINT
+{
+    page_profile_enabled = 1;
+}
+
+void gc_disable_page_profile(void) JL_NOTSAFEPOINT
+{
+    page_profile_enabled = 0;
+}
+
+void gc_page_profile_write_preamble(gc_page_profiler_serializer_t *serializer, char *data,
+                                    int osize) JL_NOTSAFEPOINT
+{
+    if (__unlikely(page_profile_enabled)) {
+        char str[GC_TYPE_STR_MAXLEN];
+        memset(str, 0, GC_TYPE_STR_MAXLEN);
+        snprintf(str, GC_TYPE_STR_MAXLEN, "\t\"address\": \"%p\",", data);
+        gc_page_serializer_write(serializer, str);
+        memset(str, 0, GC_TYPE_STR_MAXLEN);
+        snprintf(str, GC_TYPE_STR_MAXLEN, "\t\"object_size\": %d,", osize);
+        gc_page_serializer_write(serializer, str);
+        memset(str, 0, GC_TYPE_STR_MAXLEN);
+        snprintf(str, GC_TYPE_STR_MAXLEN, "\t\"objects\": [");
+        gc_page_serializer_write(serializer, str);
+    }
+}
+
+void gc_page_profile_write_epilogue(gc_page_profiler_serializer_t *serializer)
+    JL_NOTSAFEPOINT
+{
+    if (__unlikely(page_profile_enabled)) {
+        char str[GC_TYPE_STR_MAXLEN];
+        memset(str, 0, GC_TYPE_STR_MAXLEN);
+        snprintf(str, GC_TYPE_STR_MAXLEN, "\t]");
+        gc_page_serializer_write(serializer, str);
+    }
+}
+
+void gc_page_profile_write_empty_page(gc_page_profiler_serializer_t *serializer)
+    JL_NOTSAFEPOINT
+{
+    if (__unlikely(page_profile_enabled)) {
+        gc_page_serializer_write(serializer, "\t\t\"empty\",");
+    }
+}
+
+void gc_page_profile_write_garbage(gc_page_profiler_serializer_t *serializer)
+    JL_NOTSAFEPOINT
+{
+    if (__unlikely(page_profile_enabled)) {
+        gc_page_serializer_write(serializer, "\t\t\"garbage\",");
+    }
+}
+
+void gc_page_profile_write_live_obj(gc_page_profiler_serializer_t *serializer,
+                                    jl_taggedvalue_t *v) JL_NOTSAFEPOINT
+{
+    if (__unlikely(page_profile_enabled)) {
+        const char *name = jl_typeof_str(jl_valueof(v));
+        char str[GC_TYPE_STR_MAXLEN];
+        memset(str, 0, GC_TYPE_STR_MAXLEN);
+        snprintf(str, GC_TYPE_STR_MAXLEN, "\t\t\"%s\",", name);
+        gc_page_serializer_write(serializer, str);
+    }
+}
+
+void gc_page_profile_write_to_file(gc_page_profiler_serializer_t *serializer)
+    JL_NOTSAFEPOINT
+{
+    if (__unlikely(page_profile_enabled)) {
+        // write to file
+        uv_mutex_lock(&page_profile_lock);
+        ios_write(page_profile_stream, "{\n", 2);
+        ios_write(page_profile_stream, serializer->buffer, serializer->length);
+        ios_write(page_profile_stream, "}\n", 2);
+        uv_mutex_unlock(&page_profile_lock);
+    }
+}
+
+JL_DLLEXPORT void jl_gc_take_page_profile(ios_t *stream)
+{
+    gc_enable_page_profile();
+    page_profile_stream = stream;
+    jl_gc_collect(JL_GC_FULL);
+    gc_disable_page_profile();
+}
+
+#ifdef __cplusplus
+}
+#endif

src/gc-page-profiler.h

@@ -0,0 +1,42 @@
+// This file is a part of Julia. License is MIT: https://julialang.org/license
+
+#ifndef GC_PAGE_PROFILER_H
+#define GC_PAGE_PROFILER_H
+
+#include "gc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define GC_TYPE_STR_MAXLEN (512)
+
+typedef struct {
+    size_t length;
+    size_t capacity;
+    char *buffer;
+} gc_page_profiler_serializer_t;
+
+// mutex for page profile
+extern uv_mutex_t page_profile_lock;
+
+// Serializer functions
+gc_page_profiler_serializer_t gc_page_serializer_create(void) JL_NOTSAFEPOINT;
+void gc_page_serializer_init(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT;
+void gc_page_serializer_destroy(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT;
+void gc_page_serializer_write(gc_page_profiler_serializer_t *serializer, const char *str) JL_NOTSAFEPOINT;
+// Page profile functions
+void gc_page_profile_write_preamble(gc_page_profiler_serializer_t *serializer, char *data, int osize) JL_NOTSAFEPOINT;
+void gc_page_profile_write_epilogue(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT;
+void gc_page_profile_write_empty_page(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT;
+void gc_page_profile_write_garbage(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT;
+void gc_page_profile_write_live_obj(gc_page_profiler_serializer_t *serializer, jl_taggedvalue_t *v) JL_NOTSAFEPOINT;
+void gc_enable_page_profile(void) JL_NOTSAFEPOINT;
+void gc_disable_page_profile(void) JL_NOTSAFEPOINT;
+void gc_page_profile_write_to_file(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // GC_PAGE_PROFILER_H

src/gc.c

@@ -1,6 +1,7 @@
 // This file is a part of Julia. License is MIT: https://julialang.org/license
 
 #include "gc.h"
+#include "gc-page-profiler.h"
 #include "julia.h"
 #include "julia_gcext.h"
 #include "julia_assert.h"
@@ -1427,7 +1428,7 @@ STATIC_INLINE void gc_dump_page_utilization_data(void) JL_NOTSAFEPOINT
 int64_t buffered_pages = 0;
 
 // Returns pointer to terminal pointer of list rooted at *pfl.
-static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered,
+static void gc_sweep_page(gc_page_profiler_serializer_t *s, jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered,
                           jl_gc_pagemeta_t *pg, int osize) JL_NOTSAFEPOINT
 {
     char *data = pg->data;
@@ -1439,6 +1440,8 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
     }
     size_t old_nfree = pg->nfree;
     size_t nfree;
+    gc_page_serializer_init(s);
+    gc_page_profile_write_preamble(s, data, osize);
 
     int re_use_page = 1;
     int keep_as_local_buffer = 0;
@@ -1458,6 +1461,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
         }
     #endif
         nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / osize;
+        gc_page_profile_write_empty_page(s);
         goto done;
     }
     // For quick sweep, we might be able to skip the page if the page doesn't
@@ -1467,6 +1471,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
         if (!prev_sweep_full || pg->prev_nold == pg->nold) {
             freedall = 0;
             nfree = pg->nfree;
+            gc_page_profile_write_empty_page(s);
             goto done;
         }
     }
@@ -1484,12 +1489,14 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
             int bits = v->bits.gc;
             // if an object is past `lim_newpages` then we can guarantee it's garbage
             if (!gc_marked(bits) || (char*)v >= lim_newpages) {
+                gc_page_profile_write_garbage(s);
                 *pfl = v;
                 pfl = &v->next;
                 pfl_begin = (pfl_begin != NULL) ? pfl_begin : pfl;
                 pg_nfree++;
             }
             else { // marked young or old
+                gc_page_profile_write_live_obj(s, v);
                 if (current_sweep_full || bits == GC_MARKED) { // old enough
                     bits = v->bits.gc = GC_OLD; // promote
                 }
@@ -1533,6 +1540,8 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
             push_lf_back(&global_page_pool_lazily_freed, pg);
         }
     }
+    gc_page_profile_write_epilogue(s);
+    gc_page_profile_write_to_file(s);
     gc_update_page_fragmentation_data(pg);
     gc_time_count_page(freedall, pg_skpd);
     jl_ptls_t ptls = gc_all_tls_states[pg->thread_n];
@@ -1541,15 +1550,15 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
 }
 
 // the actual sweeping over all allocated pages in a memory pool
-STATIC_INLINE void gc_sweep_pool_page(jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed,
+STATIC_INLINE void gc_sweep_pool_page(gc_page_profiler_serializer_t *s, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed,
                                       jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT
 {
     int p_n = pg->pool_n;
     int t_n = pg->thread_n;
     jl_ptls_t ptls2 = gc_all_tls_states[t_n];
     jl_gc_pool_t *p = &ptls2->heap.norm_pools[p_n];
     int osize = pg->osize;
-    gc_sweep_page(p, allocd, lazily_freed, pg, osize);
+    gc_sweep_page(s, p, allocd, lazily_freed, pg, osize);
 }
 
 // sweep over all memory that is being used and not in a pool
@@ -1586,11 +1595,20 @@ void gc_sweep_wake_all(void)
     uv_mutex_unlock(&gc_threads_lock);
 }
 
+void gc_sweep_wait_for_all(void)
+{
+    jl_atomic_store(&gc_allocd_scratch, NULL);
+    while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) {
+        jl_cpu_pause();
+    }
+}
+
 void gc_sweep_pool_parallel(void)
 {
     jl_atomic_fetch_add(&gc_n_threads_sweeping, 1);
     jl_gc_page_stack_t *allocd_scratch = jl_atomic_load(&gc_allocd_scratch);
     if (allocd_scratch != NULL) {
+        gc_page_profiler_serializer_t serializer = gc_page_serializer_create();
         while (1) {
             int found_pg = 0;
             for (int t_i = 0; t_i < gc_n_threads; t_i++) {
@@ -1603,25 +1621,18 @@ void gc_sweep_pool_parallel(void)
                 if (pg == NULL) {
                     continue;
                 }
-                gc_sweep_pool_page(allocd, &ptls2->page_metadata_buffered, pg);
+                gc_sweep_pool_page(&serializer, allocd, &ptls2->page_metadata_buffered, pg);
                 found_pg = 1;
             }
             if (!found_pg) {
                 break;
             }
         }
+        gc_page_serializer_destroy(&serializer);
     }
     jl_atomic_fetch_add(&gc_n_threads_sweeping, -1);
 }
 
-void gc_sweep_wait_for_all(void)
-{
-    jl_atomic_store(&gc_allocd_scratch, NULL);
-    while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) {
-        jl_cpu_pause();
-    }
-}
-
 void gc_free_pages(void)
 {
     while (1) {
@@ -3863,6 +3874,7 @@ void jl_gc_init(void)
 {
     JL_MUTEX_INIT(&heapsnapshot_lock, "heapsnapshot_lock");
     JL_MUTEX_INIT(&finalizers_lock, "finalizers_lock");
+    uv_mutex_init(&page_profile_lock);
     uv_mutex_init(&gc_cache_lock);
     uv_mutex_init(&gc_perm_lock);
     uv_mutex_init(&gc_threads_lock);

stdlib/Profile/src/Profile.jl

@@ -16,6 +16,7 @@ public @profile,
     callers,
     init,
     take_heap_snapshot,
+    take_page_profile,
     clear_malloc_data,
     Allocs
 
@@ -1267,6 +1268,19 @@ function take_heap_snapshot(all_one::Bool=false; dir::Union{Nothing,S}=nothing)
     return take_heap_snapshot(fpath, all_one)
 end
 
+"""
+    Profile.take_page_profile(io::IOStream)
+    Profile.take_page_profile(filepath::String)
+"""
+function take_page_profile(io::IOStream)
+    Base.@_lock_ios(io, ccall(:jl_gc_take_page_profile, Cvoid, (Ptr{Cvoid},), io.handle))
+end
+function take_page_profile(filepath::String)
+    open(filepath, "w") do io
+        take_page_profile(io)
+    end
+    return filepath
+end
 
 include("Allocs.jl")
 include("precompile.jl")