Merge pull request #31 from firesim/standalone

Standalone - no more need for firesim or sudo to use marshal

centos-requirements.txt

@@ -1,4 +1,5 @@
-python34
-python34-devel
-python34-pip
+python36
+python36-devel
+python36-pip
 rsync
+libguestfs-tools

full_test.sh

@@ -7,6 +7,11 @@ LOGNAME=$(mktemp results_full_test.XXXX)
 
 echo "Running Full Test. Results available in $LOGNAME"
 
+# We pre-build to avoid potential timeouts on a fresh clone
+echo "Pre-building base workloads" | tee -a $LOGNAME
+./marshal build test/br-base.json
+./marshal build test/fedora-base.json
+
 echo "Running launch timeout test (should timeout):" | tee -a $LOGNAME
 echo "This test will reset your terminal"
 ./marshal test test/timeout-run.json | grep "timeout while running"
@@ -30,23 +35,6 @@ else
   echo "Success" | tee -a $LOGNAME
 fi
 
-# Run the specialized tests (tests that are too complicated for ./marshal
-# test)
-echo "Running clean test" | tee -a $LOGNAME
-./test/clean/test.py >> $LOGNAME 
-if [ ${PIPESTATUS[0]} != 0 ]; then
-  echo "Failure" | tee -a $LOGNAME
-  SUITE_PASS=false
-fi
-
-echo "Running incremental test" | tee -a $LOGNAME
-./test/incremental/test.py >> $LOGNAME
-if [ ${PIPESTATUS[0]} != 0 ]; then
-  echo "Failure" | tee -a $LOGNAME
-  SUITE_PASS=false
-  exit 1
-fi
-
 # Run the bulk tests (all work with the 'test' command)
 # Note the funny extended globbing, these are just lists of tests that
 # shouldn't be tested (e.g. we exclude the base configs and some specialized
@@ -77,6 +65,23 @@ else
   echo "Success" | tee -a $LOGNAME
 fi
 
+# Run the specialized tests (tests that are too complicated for ./marshal
+# test)
+echo "Running clean test" | tee -a $LOGNAME
+./test/clean/test.py >> $LOGNAME 
+if [ ${PIPESTATUS[0]} != 0 ]; then
+  echo "Failure" | tee -a $LOGNAME
+  SUITE_PASS=false
+fi
+
+echo "Running incremental test" | tee -a $LOGNAME
+./test/incremental/test.py >> $LOGNAME
+if [ ${PIPESTATUS[0]} != 0 ]; then
+  echo "Failure" | tee -a $LOGNAME
+  SUITE_PASS=false
+  exit 1
+fi
+
 echo -e "\n\nMarshal full test complete. Log at: $LOGNAME"
 if [ $SUITE_PASS = false ]; then
   echo "FAILURE: Some tests failed" | tee -a $LOGNAME

marshal

@@ -5,9 +5,10 @@ import os
 import logging
 import wlutil
 import contextlib
+import shutil
 
-if 'RISCV' not in os.environ:
-    sys.exit("Please source firesim/sourceme-manager-f1.sh first\n")
+if not shutil.which('riscv64-unknown-linux-gnu-gcc'):
+    sys.exit("No riscv toolchain detected. Please install riscv-tools.")
 
 # Delete a file but don't throw an exception if it doesn't exist
 def deleteSafe(pth):

wlutil/fedora/convert_raw.sh

@@ -27,7 +27,7 @@ DD_BS=1048576
 
 # Get the offset of the first partition within the image so we can strip it out
 # output of parted looks like "###B ####B ###B ..." we cast it to an array here
-PART_INFO=(`sudo parted -s $RAWIMG unit B print | tail -2`)
+PART_INFO=(`parted -s $RAWIMG unit B print | tail -2`)
 
 # The output at index 1 and 2 are the partition start and end byte offset. The : : -1 strips the "B" suffix.
 # Also convert to units of 1MB blocks (I'd do that above, but parted output is weird)

wlutil/install.py

@@ -5,7 +5,10 @@
 from .wlutil import *
 
 # firesim workloads directory
-fsWork = (Path(root_dir) / "../../deploy/workloads").resolve()
+try:
+    fsWork = (Path(root_dir) / "../../deploy/workloads").resolve()
+except:
+    fsWork = None
 
 readmeTxt="""This workload was generated using firesim-software. See the following config
 and workload directory for details:
@@ -18,6 +21,9 @@ def fullRel(base, target):
     return os.path.relpath(str(target), start=str(base))
 
 def installWorkload(cfgName, cfgs):
+    if fsWork == None:
+        raise RuntimeError("The install command is not supported when firesim-software is checked out standalone (i.e. not a submodule of firesim).")
+
     targetCfg = cfgs[cfgName]
     # if 'jobs' in targetCfg:
     #     raise NotImplementedError("Jobs not currently supported by the install command")

wlutil/wlutil.py

@@ -7,6 +7,8 @@
 import sys
 import collections
 import shutil
+import psutil
+import errno
 from contextlib import contextmanager
 
 wlutil_dir = os.path.normpath(os.path.dirname(__file__))
@@ -128,59 +130,40 @@ def genRunScript(command):
 
     return commandScript
 
-# XXX This isn't working with the initramfs option. Go back to requiring sudo
-# for now, I'll revisit later.
-
-# Frustratingly, the same commands don't work/exist on various platforms so we
-# need to figure out what mounting options are available to us:
-# if shutil.which('guestmount') is not None:
-#     # This is the preferred method because it doesn't require sudo
-#     @contextmanager
-#     def mountImg(imgPath, mntPath):
-#         run(['guestmount', '-a', imgPath, '-m', '/dev/sda', mntPath])
-#         try:
-#             yield mntPath
-#         finally:
-#             run(['guestunmount', mntPath])
-#
-# elif shutil.whcih('fuse-ext2') is not None:
-#     # Roughly the same as guestmount
-#     @contextmanager
-#     def mountImg(imgPath, mntPath):
-#         run(['fuse-ext2', '-o', 'rw+', imgPath, mntPath])
-#         try:
-#             yield mntPath
-#         finally:
-#             run(['fusermount', '-u', mntPath])
-#
-# elif shutil.which('mount') is not None:
-# # if True:
-#     # Should be available everywhere, requires sudo
-#     @contextmanager
-#     def mountImg(imgPath, mntPath):
-#         run(['sudo', 'mount', '-o', 'loop', imgPath, mntPath])
-#         try:
-#             yield mntPath
-#         finally:
-#             run(['sudo', 'umount', mntPath])
-#
-# else:
-#     raise ImportError("No compatible 'mount' command found")
+# This is like os.waitpid, but it works for non-child processes
+def waitpid(pid):
+    done = False
+    while not done:
+        try:
+            os.kill(pid, 0)
+        except OSError as err:
+            if err.errno == errno.ESRCH:
+                done = True
+                break
+        time.sleep(0.25)
 
 @contextmanager
 def mountImg(imgPath, mntPath):
-    run(['sudo', 'mount', '-o', 'loop', imgPath, mntPath])
+    run(['guestmount', '--pid-file', 'guestmount.pid', '-a', imgPath, '-m', '/dev/sda', mntPath])
     try:
+        with open('./guestmount.pid', 'r') as pidFile:
+            mntPid = int(pidFile.readline())
         yield mntPath
     finally:
-        run(['sudo', 'umount', mntPath])
+        run(['guestunmount', mntPath])
+        os.remove('./guestmount.pid')
+
+    # There is a race-condition in guestmount where a background task keeps
+    # modifying the image for a period after unmount. This is the documented
+    # best-practice (see man guestmount).
+    waitpid(mntPid)
 
 def toCpio(config, src, dst):
     log = logging.getLogger()
 
     with mountImg(src, mnt):
         # Fedora needs a special init in order to boot from initramfs
-        run("sudo find -print0 | sudo cpio --owner root:root --null -ov --format=newc > " + dst, shell=True, cwd=mnt)
+        run("find -print0 | cpio --owner root:root --null -ov --format=newc > " + dst, shell=True, cwd=mnt)
 
     # Ideally, the distro's themselves would provide initramfs-based versions.
     # However, having two codepaths for disk images and cpio archives
@@ -210,13 +193,13 @@ def copyImgFiles(img, files, direction):
     with mountImg(img, mnt):
         for f in files:
             # Overlays may not be owned by root, but the filesystem must be.
-            # Rsync lets us chown while copying.
+            # The FUSE mount automatically handles the chown from root to user
             # Note: shell=True because f.src is allowed to contain globs
             # Note: os.path.join can't handle overlay-style concats (e.g. join('foo/bar', '/baz') == '/baz')
             if direction == 'in':
-                run('sudo rsync -a --chown=root:root ' + f.src + " " + os.path.normpath(mnt + f.dst), shell=True)
+                run('cp -a ' + f.src + " " + os.path.normpath(mnt + f.dst), shell=True)
             elif direction == 'out':
                 uid = os.getuid()
-                run('sudo rsync -a --chown=' + str(uid) + ':' + str(uid) + ' ' + os.path.normpath(mnt + f.src) + " " + f.dst, shell=True)
+                run('cp -a ' + os.path.normpath(mnt + f.src) + " " + f.dst, shell=True)
             else:
                 raise ValueError("direction option must be either 'in' or 'out'")