adding locking to prevent two parallel builds of the same installation directory

easybuild/framework/easyblock.py

@@ -2943,20 +2943,42 @@ def run_all_steps(self, run_test_cases):
 
         print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent)
         trace_msg("installation prefix: %s" % self.installdir)
-        try:
-            for (step_name, descr, step_methods, skippable) in steps:
-                if self._skip_step(step_name, skippable):
-                    print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent)
-                else:
-                    if self.dry_run:
-                        self.dry_run_msg("%s... [DRY RUN]\n", descr)
-                    else:
-                        print_msg("%s..." % descr, log=self.log, silent=self.silent)
-                    self.current_step = step_name
-                    self.run_step(step_name, step_methods)
 
-        except StopException:
-            pass
+        lockpath = build_option('lockpath') or build_path()
+        if not os.path.exists(lockpath):
+            mkdir(lockpath)
+        lockfile_name = os.path.join(lockpath, ".%s.lock" % self.installdir.replace('/','_'))
+        if os.path.exists(lockfile_name):
+            if build_option('wait_on_lock'):
+                while os.path.exists(lockfile_name):
+                    print_msg("Lock file %s exists. Waiting 60 seconds." % lockfile_name, silent=self.silent)
+                    time.sleep(60)
+            else:
+                print_msg("Build aborted. Lock file %s exists." % lockfile_name, silent=self.silent)
+                return False
+        else:
+            try:
+                # create a new lock file
+                print_msg("Creating lock file %s" % lockfile_name)
+                f = open(lockfile_name, "w+")
+                f.close()
+
+                for (step_name, descr, step_methods, skippable) in steps:
+                    if self._skip_step(step_name, skippable):
+                        print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent)
+                    else:
+                        if self.dry_run:
+                            self.dry_run_msg("%s... [DRY RUN]\n", descr)
+                        else:
+                            print_msg("%s..." % descr, log=self.log, silent=self.silent)
+                        self.current_step = step_name
+                        self.run_step(step_name, step_methods)
+
+            except StopException:
+                pass
+            finally:
+                print_msg("Removing lock file %s" % lockfile_name, silent=self.silent)
+                os.remove(lockfile_name)
 
         # return True for successfull build (or stopped build)
         return True

easybuild/tools/config.py

@@ -162,6 +162,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'job_output_dir',
         'job_polling_interval',
         'job_target_resource',
+        'lockpath',
         'modules_footer',
         'modules_header',
         'mpi_cmd_template',
@@ -221,6 +222,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'use_f90cache',
         'use_existing_modules',
         'set_default_module',
+        'wait_on_lock',
     ],
     True: [
         'cleanup_builddir',

easybuild/tools/options.py

@@ -273,6 +273,7 @@ def basic_options(self):
                       "and skipping check for OS dependencies", None, 'store_true', False, 'f'),
             'job': ("Submit the build as a job", None, 'store_true', False),
             'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'),
+            'lockpath': ("Specifies which path should be used to store lock files", None, 'store_or_None', None),
             'missing-modules': ("Print list of missing modules for dependencies of specified easyconfigs",
                                 None, 'store_true', False, 'M'),
             'only-blocks': ("Only build listed blocks", 'strlist', 'extend', None, 'b', {'metavar': 'BLOCKS'}),
@@ -441,6 +442,8 @@ def override_options(self):
                                      None, 'store_true', False),
             'verify-easyconfig-filenames': ("Verify whether filename of specified easyconfigs matches with contents",
                                             None, 'store_true', False),
+            'wait-on-lock': ("Wait until lock file is removed when a lock if found",
+                             None, 'store_true', False),
             'zip-logs': ("Zip logs that are copied to install directory, using specified command",
                          None, 'store_or_None', 'gzip'),