Adding delayed start and max measurement

Adding APEX_START_DELAY_SECONDS option and APEX_MAX_DURATION_SECONDS
to specify a delayed start of measurement, and a max length to measure.
The max length does not include delayed time.  So a delay of 1
seconds and a max length of 2 seconds would record seconds 1-3 of an
execution.

src/apex/apex.cpp

@@ -426,6 +426,7 @@ uint64_t init(const char * thread_name, uint64_t comm_rank,
             instance->listeners[i]->on_startup(data);
         }
     }
+    handle_delayed_start();
     // start accepting requests, now that all listeners are started
     _initialized = true;
 #if APEX_HAVE_PROC

src/apex/apex_types.h

@@ -329,6 +329,8 @@ inline unsigned int sc_nprocessors_onln()
     macro (APEX_JUPYTER_SUPPORT, use_jupyter_support, int, false) \
     macro (APEX_KOKKOS_VERBOSE, use_kokkos_verbose, bool, false) \
     macro (APEX_KOKKOS_TUNING, use_kokkos_tuning, bool, true) \
+    macro (APEX_START_DELAY_SECONDS, start_delay_seconds, int, 0) \
+    macro (APEX_MAX_DURATION_SECONDS, max_duration_seconds, int, 0) \
 
 #define FOREACH_APEX_FLOAT_OPTION(macro) \
     macro (APEX_SCATTERPLOT_FRACTION, scatterplot_fraction, double, 0.01) \

src/apex/utils.cpp

@@ -35,6 +35,8 @@
 #include <string>
 #include <vector>
 #include <mutex>
+#include <thread>
+#include <chrono>
 
 namespace apex {
 
@@ -530,6 +532,38 @@ uint64_t test_for_MPI_comm_size(uint64_t commsize) {
     return commsize;
 }
 
+/* Thread that sleeps for a bit then enables data collection */
+void threaded_start_delay() {
+    std::this_thread::sleep_for(
+        std::chrono::seconds(
+        apex_options::start_delay_seconds()));
+    // enable data collection
+    apex_options::suspend(false);
+}
+
+/* Thread that sleeps for a bit then disables data collection */
+void threaded_stop_recording() {
+    std::this_thread::sleep_for(
+        std::chrono::seconds(
+        apex_options::start_delay_seconds() +
+        apex_options::max_duration_seconds()));
+    // disable data collection
+    apex_options::suspend(true);
+}
+
+void handle_delayed_start() {
+    if (apex_options::start_delay_seconds() > 0) {
+        // disable data collection
+        apex_options::suspend(true);
+        static std::thread delayed_start(threaded_start_delay);
+        delayed_start.detach();
+    }
+    if (apex_options::max_duration_seconds() > 0) {
+        static std::thread stop_recording(threaded_stop_recording);
+        stop_recording.detach();
+    }
+}
+
 std::string activity_to_string(apex_async_activity_t activity) {
     static std::string kernel{"Compute"};
     static std::string memory{"Memory"};

src/apex/utils.hpp

@@ -256,6 +256,7 @@ inline char filesystem_separator()
 
 uint64_t test_for_MPI_comm_rank(uint64_t commrank);
 uint64_t test_for_MPI_comm_size(uint64_t commsize);
+void handle_delayed_start(void);
 
 std::string activity_to_string(apex_async_activity_t activity);