Add runtime option to eagerly run GC

Summary: Add a new runtime option called "Eval.EagerGCProbability". If set to a value
greater than zero, a garbage collection will be eagerly run at every
allocation or deallocation with probability 1/N. Eval.EnableGC must be set to 1
for this option to do anything. To avoid overhead in production, this option is
only available in debug builds.

This is implemented by doing a random coin-flip with the specified probability
at every allocation or deallocation. If the coin-flip succeeds, a new surprise
flag is set. A garbage collection is then done at the next surprise flag
check. This is safer than attempting the garbage collection in the middle of the
allocation. In the future, the surprise check flag will be used when we need to
run a GC for real.

Right now enabling this option often causes asserts to be triggered, because the
heap isn't always in a fully consistent state even during the surprise flag
checks. This is being worked on.

Reviewed By: @edwinsmith

Differential Revision: D2315369

hphp/runtime/base/memory-manager-inl.h

@@ -237,6 +237,9 @@ inline uint32_t MemoryManager::smallSizeClass(uint32_t reqBytes) {
 inline void* MemoryManager::mallocSmallIndex(size_t index, uint32_t bytes) {
   assert(index < kNumSmallSizes);
   assert(bytes <= kSmallIndex2Size[index]);
+
+  if (debug) eagerGCCheck();
+
   m_stats.usage += bytes;
 
   void *p = m_freelists[index].maybePop();
@@ -265,6 +268,8 @@ inline void MemoryManager::freeSmallIndex(void* ptr, size_t index,
     return freeBigSize(ptr, bytes);
   }
 
+  if (debug) eagerGCCheck();
+
   FTRACE(3, "freeSmallSize({}, {}), freelist {}\n", ptr, bytes, index);
 
   m_freelists[index].push(ptr, bytes);
@@ -282,6 +287,8 @@ inline void MemoryManager::freeSmallSize(void* ptr, uint32_t bytes) {
     return freeBigSize(ptr, bytes);
   }
 
+  if (debug) eagerGCCheck();
+
   auto const i = smallSize2Index(bytes);
   FTRACE(3, "freeSmallSize({}, {}), freelist {}\n", ptr, bytes, i);
 
@@ -294,6 +301,7 @@ inline void MemoryManager::freeSmallSize(void* ptr, uint32_t bytes) {
 
 ALWAYS_INLINE
 void MemoryManager::freeBigSize(void* vp, size_t bytes) {
+  if (debug) eagerGCCheck();
   m_stats.usage -= bytes;
   // Since we account for these direct allocations in our usage and adjust for
   // them on allocation, we also need to adjust for them negatively on free.

hphp/runtime/base/memory-manager.cpp

@@ -35,6 +35,7 @@
 #include "hphp/util/process.h"
 #include "hphp/util/trace.h"
 
+#include <folly/Random.h>
 #include <folly/ScopeGuard.h>
 #include "hphp/runtime/base/memory-manager-defs.h"
 
@@ -604,6 +605,7 @@ inline void* MemoryManager::realloc(void* ptr, size_t nbytes) {
     return newmem;
   }
   // Ok, it's a big allocation.
+  if (debug) eagerGCCheck();
   auto block = m_heap.resizeBig(ptr, nbytes);
   refreshStats();
   return block.ptr;
@@ -917,6 +919,8 @@ MemBlock MemoryManager::mallocBigSize<false>(size_t);
 
 template<bool callerSavesActualSize> NEVER_INLINE
 MemBlock MemoryManager::mallocBigSize(size_t bytes) {
+  if (debug) eagerGCCheck();
+
   auto block = m_heap.allocBig(bytes, HeaderKind::BigObj);
   auto szOut = block.size;
 #ifdef USE_JEMALLOC
@@ -937,6 +941,7 @@ MemBlock MemoryManager::mallocBigSize(size_t bytes) {
 
 NEVER_INLINE
 void* MemoryManager::callocBig(size_t totalbytes) {
+  if (debug) eagerGCCheck();
   assert(totalbytes > 0);
   auto block = m_heap.callocBig(totalbytes);
   updateBigStats();
@@ -1108,6 +1113,14 @@ void MemoryManager::requestShutdown() {
   profctx = ReqProfContext{};
 }
 
+void MemoryManager::eagerGCCheck() {
+  if (RuntimeOption::EvalEagerGCProbability > 0 &&
+      !g_context.isNull() &&
+      folly::Random::oneIn(RuntimeOption::EvalEagerGCProbability)) {
+    setSurpriseFlag(PendingGCFlag);
+  }
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 
 void BigHeap::reset() {

hphp/runtime/base/memory-manager.h

@@ -979,6 +979,8 @@ struct MemoryManager {
   void dropRootMaps();
   void deleteRootMaps();
 
+  void eagerGCCheck();
+
   template <typename T>
   typename std::enable_if<
     std::is_base_of<ResourceData,T>::value,

hphp/runtime/base/runtime-option.h

@@ -551,6 +551,11 @@ class RuntimeOption {
   F(bool, RandomHotFuncs,              false)                           \
   F(bool, CheckHeapOnAlloc,            false)                           \
   F(bool, EnableGC,                    false)                           \
+  /*
+    Run GC on every allocation/deallocation with probability 1/N (0 to
+    disable). Requires EnableGC=true with debug build.
+  */                                                                    \
+  F(uint32_t, EagerGCProbability,   0)                                  \
   F(bool, DisableSomeRepoAuthNotices,  true)                            \
   F(uint32_t, InitialNamedEntityTableSize,  30000)                      \
   F(uint32_t, InitialStaticStringTableSize,                             \

hphp/runtime/base/surprise-flags.h

@@ -46,12 +46,20 @@ enum SurpriseFlag : size_t {
   CPUTimedOutFlag      = 1ul << 58,
   IntervalTimerFlag    = 1ul << 59,
 
+  /* Set if a GC should be run at the next safe point. */
+  PendingGCFlag        = 1ul << 60,
+
   /*
    * Flags that shouldn't be cleared by fetchAndClearSurpriseFlags, because
    * fetchAndClearSurpriseFlags is only supposed to touch flags related to
    * PHP-visible signals/exceptions and resource limits.
    */
-  ResourceFlags = MemExceededFlag | TimedOutFlag | CPUTimedOutFlag,
+  ResourceFlags =
+    MemExceededFlag |
+    TimedOutFlag |
+    CPUTimedOutFlag |
+    PendingGCFlag,
+
   StickyFlags =
     AsyncEventHookFlag |
     DebuggerHookFlag |

hphp/runtime/base/thread-info.cpp

@@ -178,6 +178,7 @@ size_t check_request_surprise() {
   auto const do_signaled = flags & SignaledFlag;
   auto const do_cpuTimedOut =
     (flags & CPUTimedOutFlag) && !p.getDebuggerAttached();
+  auto const do_GC = flags & PendingGCFlag;
 
   // Start with any pending exception that might be on the thread.
   auto pendingException = info.m_pendingException;
@@ -207,6 +208,9 @@ size_t check_request_surprise() {
       pendingException = generate_memory_exceeded_exception();
     }
   }
+  if (do_GC) {
+    MM().collect();
+  }
   if (do_signaled) {
     HHVM_FN(pcntl_signal_dispatch)();
   }