[virt] Significant updates for system calls, signals, and interval ti…

…mers.

  - Switch to using stacked syscall state in order to support system
    calls within signal handlers and auto-restarting syscalls.
  - Add additional syscall information, including whether it has been
    interrupted and what the effective (simulated) call is.
  - Handle the behavior of SYS_rt_sigreturn, which never exits.
  - Never call SyscallExit in the signal callback.
  - In the post-patch for nanosleep, assume EINTR (a signal
    interruption) if the current phase is earlier than the wakeup phase.

[scheduler] Change notifySleepEnd to return the canceled wakeup phase.

[tests] Update/extend the interval-timer test program.

[misc] Make sure lineSize is read from the zsim config before
       initializing the scheduler (which depends on it via the interval
       timer).

misc/testProgs/test-timers.cpp

@@ -141,6 +141,36 @@ static void Setitimer(int which, const struct itimerval *new_value, struct itime
     }
 }
 
+// Test nanosleep
+static void testNanosleep() {
+    diff_.clear();
+    struct timespec newval, oldval;
+    newval.tv_sec = 0;
+    newval.tv_nsec = 5000000;  // 0.5 seconds
+    int res = nanosleep(&newval, &oldval);
+    if (res != 0) {
+        FAIL("Nanosleep failed", "");
+    }
+    newval.tv_sec = 1;
+    begin_ = std::chrono::steady_clock::now();
+    alarm(1);
+    res = nanosleep(&newval, &oldval);
+    if (diff_.empty()) {
+        FAIL("Didn't see SIGALRM", "");
+    }
+    if (res != -1) {
+        FAIL("Expected nanosleep to fail", "");
+    }
+    if (notClose(diff_.front(), 1.0)) {
+        FAIL("Expected 1 second (got %f)", diff_.front());
+    }
+    double remain = oldval.tv_sec;
+    remain += 1e-9 * oldval.tv_nsec;
+    if (notClose(remain, 0.5)) {
+        FAIL("Expected remaining time of 0.5 seconds (got %f)", remain);
+    }
+}
+
 // Test a single alarm() call with spinning instead of sleeping
 static void testAlarmSimple() {
     diff_.clear();
@@ -160,7 +190,7 @@ static void testAlarmSimple() {
         FAIL("Expected to see 1 signal (saw %lu)", diff_.size());
     }
     if (notClose(diff_.front(), 1.0)) {
-        FAIL("Expected 1 second (got %f)", diff_);
+        FAIL("Expected 1 second (got %f)", diff_.front());
     }
     alarm(0);
 }
@@ -178,13 +208,16 @@ static void testAlarm() {
     if (notClose(prior, 10)) {
         FAIL("Expected 10 seconds (got %u)", prior);
     }
-    sleep(2);
+    struct timespec amt;
+    amt.tv_sec = 2;
+    amt.tv_nsec = 0;
+    nanosleep(&amt, nullptr);   // Sleep 2 seconds
     if (diff_.empty()) {
         FAIL("Didn't see SIGALRM", "");
     }
     checkOneSignal();
     if (notClose(diff_.front(), 1.0)) {
-        FAIL("Expected 1 second (got %f)", diff_);
+        FAIL("Expected 1 second (got %f)", diff_.front());
     }
     // Test 2: Busy work while waiting
     diff_.clear();
@@ -242,12 +275,15 @@ static void testSetReal() {
     val.it_interval.tv_usec = 0;
     val.it_value.tv_sec = 2;
     val.it_value.tv_usec = 0;
+    struct timespec nsleep;
+    nsleep.tv_sec = 3;
+    nsleep.tv_nsec = 0;
     Setitimer(ITIMER_REAL, &val, nullptr);
 
     for (int i = 0; i < 2; i++) {
         diff_.clear();
         begin_ = std::chrono::steady_clock::now();
-        sleep(3);
+        nanosleep(&nsleep, nullptr);   // Sleep 3 seconds
         if (diff_.empty()) {
             FAIL("Didn't see SIGALRM", "");
         }
@@ -280,7 +316,18 @@ static void testSetVirtualProf(int which) {
     // Test 1: One active thread (the busy thread)
     diff_.clear();
     begin_ = std::chrono::steady_clock::now();
-    sleep(4); // A reasonable system should have >= 2 signals
+
+    // Sleep for 4 seconds. A reasonable system should have >= 2 signals
+    struct timespec nsleep, rem;
+    nsleep.tv_sec = 4;
+    nsleep.tv_nsec = 0;
+    int res = nanosleep(&nsleep, &rem);
+    while (res != 0) {
+        //printf("   ns remain: %lu sec, %lu nsec\n", rem.tv_sec, rem.tv_nsec);
+        nsleep = rem;
+        res = nanosleep(&nsleep, &rem);
+    }
+
     Setitimer(which, &zero, nullptr);
     if (diff_.empty()) {
         FAIL("Didn't see SIGVTALRM/SIGPROF", "");
@@ -324,6 +371,8 @@ int main() {
     Signal(SIGVTALRM, handler);
     Signal(SIGPROF, handler);
     std::thread worker(busyWork, 0);  // Run a continuous busy thread
+    printf("Tests begin here\n");
+    testNanosleep();
     testAlarmSimple();
     testAlarm();
     testGetItimer();

src/init.cpp

@@ -937,14 +937,19 @@ void SimInit(const char* configFile, const char* outputDir, uint32_t shmid) {
 
     zinfo->eventQueue = new EventQueue(); //must be instantiated before the memory hierarchy
 
+    //Process tree needs this initialized, even though it is part of the memory hierarchy
+    //The scheduler also needs the max number of processes which we derive from here (e.g., see process_tree.cpp)
+    zinfo->lineSize = config.get<uint32_t>("sys.lineSize", 64);
+    assert(zinfo->lineSize > 0);
+
     if (!zinfo->traceDriven) {
         //Build the scheduler
         uint32_t parallelism = config.get<uint32_t>("sim.parallelism", 2*sysconf(_SC_NPROCESSORS_ONLN));
         if (parallelism < zinfo->numCores) info("Limiting concurrent threads to %d", parallelism);
         assert(parallelism > 0); //jeez...
 
         uint32_t schedQuantum = config.get<uint32_t>("sim.schedQuantum", 10000); //phases
-        zinfo->sched = new Scheduler(EndOfPhaseActions, parallelism, zinfo->numCores, schedQuantum);
+        zinfo->sched = new Scheduler(EndOfPhaseActions, parallelism, zinfo->numCores, schedQuantum, zinfo->lineSize);
     } else {
         zinfo->sched = nullptr;
     }
@@ -962,10 +967,6 @@ void SimInit(const char* configFile, const char* outputDir, uint32_t shmid) {
     allCoreStats->init("core", "Core stats");
     zinfo->rootStat->append(allCoreStats);
 
-    //Process tree needs this initialized, even though it is part of the memory hierarchy
-    zinfo->lineSize = config.get<uint32_t>("sys.lineSize", 64);
-    assert(zinfo->lineSize > 0);
-
     //Port virtualization
     for (uint32_t i = 0; i < MAX_PORT_DOMAINS; i++) zinfo->portVirt[i] = new PortVirtualizer();
 

src/scheduler.h

@@ -169,8 +169,8 @@ class Scheduler : public GlobAlloc, public Callee {
         inline uint32_t getTid(uint32_t gid) const {return gid & 0x0FFFF;}
 
     public:
-        Scheduler(void (*_atSyncFunc)(void), uint32_t _parallelThreads, uint32_t _numCores, uint32_t _schedQuantum) :
-            atSyncFunc(_atSyncFunc), bar(_parallelThreads, this), numCores(_numCores), schedQuantum(_schedQuantum), rnd(0x5C73D9134), intervalTimer((uint32_t)zinfo->lineSize)
+        Scheduler(void (*_atSyncFunc)(void), uint32_t _parallelThreads, uint32_t _numCores, uint32_t _schedQuantum, uint32_t _maxProcesses) :
+            atSyncFunc(_atSyncFunc), bar(_parallelThreads, this), numCores(_numCores), schedQuantum(_schedQuantum), rnd(0x5C73D9134), intervalTimer(_maxProcesses)
         {
             contexts.resize(numCores);
             for (uint32_t i = 0; i < numCores; i++) {
@@ -486,7 +486,8 @@ class Scheduler : public GlobAlloc, public Callee {
             return res;
         }
 
-        void notifySleepEnd(uint32_t pid, uint32_t tid) {
+        uint64_t notifySleepEnd(uint32_t pid, uint32_t tid) {
+            uint64_t wakeupPhase;
             futex_lock(&schedLock);
             uint32_t gid = getGid(pid, tid);
             ThreadInfo* th = gidMap[gid];
@@ -499,7 +500,9 @@ class Scheduler : public GlobAlloc, public Callee {
                 sleepQueue.remove(th);
                 th->state = BLOCKED;
             }
+            wakeupPhase = th->wakeupPhase;
             futex_unlock(&schedLock);
+            return wakeupPhase;
         }
 
         void printThreadState(uint32_t pid, uint32_t tid) {

src/virt/common.h

@@ -39,6 +39,7 @@ struct PrePatchArgs {
     CONTEXT* ctxt;
     SYSCALL_STANDARD std;
     const char* patchRoot;
+    long *actualSyscall;  //Set this if the prepatch changes the syscall number
     bool isNopThread;
 };
 

src/virt/interval_timer.cpp

@@ -82,8 +82,6 @@ void IntervalTimer::phaseTick() {
             assert(si->phase == curPhase);
             trace(Sched, "Sending delayed signal %d to pid %d at phase %lu", si->sig, si->osPid, curPhase);
 
-            //XXX TODO: May need to unblock sleeping threads first
-
             //Note that SYS_tgkill could be used instead to target a specific thread
             syscall(SYS_kill, si->osPid, si->sig);  // XXX errno is not thread-safe for pin tools. Check
 

src/virt/time.cpp

@@ -217,6 +217,8 @@ PostPatchFn PatchNanosleep(PrePatchArgs args) {
     struct timespec* rem = (struct timespec*) PIN_GetSyscallArgument(ctxt, std, isClock? 3 : 1);
 
     // Turn this into a non-timed FUTEX_WAIT syscall
+    trace(TimeVirt, "[%d] Changing actual syscall to SYS_futex (%d)", args.tid, SYS_futex);
+    *(args.actualSyscall) = SYS_futex;
     PIN_SetSyscallNumber(ctxt, std, SYS_futex);
     PIN_SetSyscallArgument(ctxt, std, 0, (ADDRINT)futexWord);
     PIN_SetSyscallArgument(ctxt, std, 1, (ADDRINT)FUTEX_WAIT);
@@ -233,8 +235,15 @@ PostPatchFn PatchNanosleep(PrePatchArgs args) {
             trace(TimeVirt, "[%d] Post-patching SYS_nanosleep", args.tid);
         }
 
+        //TODO: Doesn't clock_nanosleep directly return the (positive) errno?
         int res = (int)(-PIN_GetSyscallNumber(ctxt, std));
-        if (res == EWOULDBLOCK) {
+        if (wakeupPhase > zinfo->numPhases) {
+            // Assume this is due to a signal interruption (rather than zsim bug)
+            trace(TimeVirt, "Sleep woke up early (%lu phases); assuming a signal and setting EINTR",
+                wakeupPhase - zinfo->numPhases);
+            res = EINTR;
+            PIN_SetSyscallNumber(ctxt, std, -EINTR);
+        } else if (res == EWOULDBLOCK) {
             trace(TimeVirt, "Fixing EWOULDBLOCK --> 0");
             PIN_SetSyscallNumber(ctxt, std, 0);  // this is fine, you just called a very very short sleep
         } else if (res == EINTR) {
@@ -254,7 +263,7 @@ PostPatchFn PatchNanosleep(PrePatchArgs args) {
         if (rem) {
             if (res == EINTR) {
                 assert(wakeupPhase >= zinfo->numPhases);  // o/w why is this EINTR...
-                uint64_t remainingCycles = wakeupPhase - zinfo->numPhases;
+                uint64_t remainingCycles = zinfo->phaseLength * (wakeupPhase - zinfo->numPhases);
                 uint64_t remainingNsecs = remainingCycles*1000/zinfo->freqMHz;
                 rem->tv_sec = remainingNsecs/1000000000;
                 rem->tv_nsec = remainingNsecs % 1000000000;
@@ -380,7 +389,9 @@ PostPatchFn PatchSetitimerSyscall(PrePatchArgs args) {
 
         //Set oldVal and free memory
         ADDRINT arg2 = PIN_GetSyscallArgument(ctxt, std, 2);
-        PIN_SafeCopy((void *)arg2, oldVal, sizeof(struct itimerval));
+        if (arg2 != 0) {
+            PIN_SafeCopy((void *)arg2, oldVal, sizeof(struct itimerval));
+        }
         delete newVal;
         delete oldVal;
 

src/virt/virt.cpp

@@ -71,13 +71,13 @@ void VirtInit() {
 
 
 // Dispatch methods
-void VirtSyscallEnter(THREADID tid, CONTEXT *ctxt, SYSCALL_STANDARD std, const char* patchRoot, bool isNopThread) {
+void VirtSyscallEnter(THREADID tid, CONTEXT *ctxt, SYSCALL_STANDARD std, const char* patchRoot, long *actualSyscall, bool isNopThread) {
     uint32_t syscall = PIN_GetSyscallNumber(ctxt, std);
     if (syscall >= MAX_SYSCALLS) {
         warn("syscall %d out of range", syscall);
         postPatchFunctions[tid] = NullPostPatch;
     } else {
-        postPatchFunctions[tid] = prePatchFunctions[syscall]({tid, ctxt, std, patchRoot, isNopThread});
+        postPatchFunctions[tid] = prePatchFunctions[syscall]({tid, ctxt, std, patchRoot, actualSyscall, isNopThread});
     }
 }
 

src/virt/virt.h

@@ -38,7 +38,7 @@ enum PostPatchAction {
 };
 
 void VirtInit();  // per-process, not global
-void VirtSyscallEnter(THREADID tid, CONTEXT *ctxt, SYSCALL_STANDARD std, const char* patchRoot, bool isNopThread);
+void VirtSyscallEnter(THREADID tid, CONTEXT *ctxt, SYSCALL_STANDARD std, const char* patchRoot, long *actualSyscall, bool isNopThread);
 PostPatchAction VirtSyscallExit(THREADID tid, CONTEXT *ctxt, SYSCALL_STANDARD std);
 
 // VDSO / external virt functions