Merge pull request ofiwg#5966 from nikhilnanal/feat_mpichsuite

Jenkins-CI: enabled spawn tests

contrib/intel/jenkins/Jenkinsfile

@@ -44,48 +44,47 @@ pipeline {
 
         stage ('build-shmem') {
             steps {
-              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) {
+              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']){
                 sh """
                 python3.7  contrib/intel/jenkins/build.py 'shmem' --ofi_build_mode='dbg'
                 echo 'shmem benchmarks built successfully'
                 """
-                }
               }
-          }
-
-        stage ('build OMPI_bm') {
-              steps {
-              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) {
-                  sh """
-                  python3.7 contrib/intel/jenkins/build.py 'ompi_benchmarks' --ofi_build_mode='dbg'
-                  echo 'mpi benchmarks with ompi - built successfully'
-                 """
-                }
+            }
+        }
+        stage('build MPICH_bm') {
+            steps {
+              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']){
+                sh """
+                python3.7 contrib/intel/jenkins/build.py 'mpich_benchmarks' --ofi_build_mode='dbg'
+                echo "mpi benchmarks with mpich - built successfully"
+                """
               }
-          }
-
-    stage('build IMPI_bm') {
-        steps {
-          withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) {
+            }
+        }
+        stage('build IMPI_bm') {
+            steps {
+              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']){
                 sh """
                 python3.7 contrib/intel/jenkins/build.py 'impi_benchmarks' --ofi_build_mode='dbg'
                 echo 'mpi benchmarks with impi - built successfully'
                 """
+              }
             }
-          }
-      }  
-
-    stage('build MPICH_bm') {
-        steps {
-          withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) {
+        }  
+
+        stage ('build OMPI_bm') {
+            steps {
+              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']){
                 sh """
-                python3.7 contrib/intel/jenkins/build.py 'mpich_benchmarks' --ofi_build_mode='dbg'
-                echo "mpi benchmarks with mpich - built successfully"
+                python3.7 contrib/intel/jenkins/build.py 'ompi_benchmarks' --ofi_build_mode='dbg'
+                echo 'mpi benchmarks with ompi - built successfully'
                 """
               }
             }
         }
-   stage('parallel-tests') {
+
+        stage('parallel-tests') {
             parallel { 
                  stage('eth-tcp-dbg') {
                     agent {node {label 'eth'}}

contrib/intel/jenkins/build.py

@@ -10,6 +10,7 @@
 import shlex
 import common
 import re
+import shutil
 
 def build_libfabric(libfab_install_path, mode):
 
@@ -138,13 +139,30 @@ def build_mpi(mpi, mpisrc, mpi_install_path, libfab_install_path,  ofi_build_mod
         cmd.append("--enable-fortran=no")
         cmd.append("--with-device=ch4:ofi")
         cmd.append("--enable-ch4-direct=netmod")
-
 
     configure_cmd = shlex.split(" ".join(cmd))
     common.run_command(configure_cmd)
     common.run_command(["make", "clean"])
     common.run_command(["make", "install", "-j32"])
 
+def build_mpich_suite(mpi, mpi_install_path, libfab_install_path):
+
+    mpich_suite_path = '{}/test/'.format(ci_site_config.mpich_src)
+    mpichsuite_installpath= "{}/mpichsuite/test".format(mpi_install_path)
+    pwd = os.getcwd()
+    if (mpi == 'impi'):
+        os.chdir("{}/mpi".format(mpich_suite_path))
+        cmd = ["./configure", "--with-mpi={}/intel64" \
+               .format(ci_site_config.impi_root)]
+
+        configure_cmd = shlex.split(" ".join(cmd))
+        common.run_command(configure_cmd)
+        common.run_command(["make", "all","-j32"])
+        shutil.copytree(mpich_suite_path, mpichsuite_installpath)
+        common.run_command(["make", "distclean"])
+        os.chdir(pwd)
+
+
 
 def build_stress_bm(mpi, mpi_install_path, libfab_install_path):
 
@@ -247,7 +265,9 @@ def build_osu_bm(mpi, mpi_install_path, libfab_install_path):
                      else ci_site_config.ompi_src
             # only need to build ompi or mpich, impi is available as binary
             build_mpi(mpi, mpisrc, mpi_install_path, install_path, ofi_build_mode)
-
+
+	# build mpich_test_suite
+        build_mpich_suite(mpi, mpi_install_path, install_path)
         # run stress and osu benchmarks for all mpitypes
         build_stress_bm(mpi, mpi_install_path, install_path)
         build_osu_bm(mpi, mpi_install_path, install_path)

contrib/intel/jenkins/run.py

@@ -75,7 +75,19 @@ def intel_mpi_benchmark(core, hosts, mpi, mode, util=None):
         print("skipping {} as execute condition fails"\
                     .format(imb_test.testname))
     print("----------------------------------------------------------------------------------------\n")
-
+
+#mpich_test_suite
+def mpich_test_suite(core, hosts, mpi, mode, util=None):
+    mpich_tests = tests.MpichTestSuite(jobname=jbname,buildno=bno,\
+                  testname="MpichTestSuite",core_prov=core, fabric=fab,\
+                  mpitype=mpi, hosts=hosts, ofi_build_mode=mode, \
+                  util_prov=util)
+    if (mpich_tests.execute_condn == True and \
+        mpich_tests.mpi_gen_execute_condn == True):
+        print("Running mpich test suite: Spawn coll, comm, dt Tests for {}-{}-{}-{}".format(core, util, fab, mpi))
+        os.environ["MPITEST_RETURN_WITH_CODE"] = "1"
+        mpich_tests.execute_cmd("spawn")
+
 #mpi_stress benchmark tests
 def mpistress_benchmark(core, hosts, mpi, mode, util=None):
 

contrib/intel/jenkins/runtests.py

@@ -50,6 +50,7 @@
         run.fabtests(args_core, hosts, ofi_build_mode)
         run.shmemtest(args_core, hosts, ofi_build_mode)
         for mpi in mpilist:
+            run.mpich_test_suite(args_core, hosts, mpi, ofi_build_mode)
             run.intel_mpi_benchmark(args_core, hosts, mpi, ofi_build_mode)   
             run.mpistress_benchmark(args_core, hosts, mpi, ofi_build_mode)
             run.osu_benchmark(args_core, hosts, mpi, ofi_build_mode)  
@@ -58,10 +59,13 @@
         run.fabtests(args_core, hosts, ofi_build_mode, util=args_util)
         run.shmemtest(args_core, hosts, ofi_build_mode, util=args_util)
         for mpi in mpilist:
+            run.mpich_test_suite(args_core, hosts, mpi, ofi_build_mode, \
+                                 util=args_util)
+
             run.intel_mpi_benchmark(args_core, hosts, mpi, ofi_build_mode, \
-                                        util=args_util,)
+                                    util=args_util)
             run.mpistress_benchmark(args_core, hosts, mpi, ofi_build_mode, \
-                                            util=args_util)
+                                    util=args_util)
             run.osu_benchmark(args_core, hosts, mpi, ofi_build_mode, \
                                              util=args_util)
 else:

contrib/intel/jenkins/tests.py

@@ -207,7 +207,6 @@ def __init__(self, jobname, buildno, testname, core_prov, fabric,
                          fabric, hosts, ofi_build_mode, util_prov)
         self.mpi = mpitype
 
-
     @property
     def cmd(self):
         if (self.mpi == "impi" or self.mpi == "mpich"):
@@ -358,7 +357,76 @@ def execute_cmd(self):
             outputcmd = shlex.split(command + self.rma.imb_cmd)
             common.run_command(outputcmd)
 
+class MpichTestSuite(MpiTests):
+
+    def __init__(self, jobname, buildno, testname, core_prov, fabric, 
+		     mpitype, hosts, ofi_build_mode, util_prov=None):
+            super().__init__(jobname, buildno, testname, core_prov, fabric,
+			     mpitype,  hosts, ofi_build_mode, util_prov)
+            self.mpichsuitepath =  "{}/{}/mpichsuite/test/mpi/" \
+                                   .format(self.libfab_installpath, self.mpi)
+            self.pwd = os.getcwd()
+
+    def testgroup(self, testgroupname):
 
+        testpath = "{}/{}".format(self.mpichsuitepath, testgroupname)
+        tests = []
+        with open("{}/testlist".format(testpath)) as file:
+            for line in file:
+                if(line[0] != '#' and  line[0] != '\n'):
+                    tests.append((line.rstrip('\n')).split(' '))
+
+        return tests
+
+    def options(self, nprocs, timeout=None):
+        if (self.mpi == "impi" or self.mpi == "mpich"):
+            if (self.mpi == "impi"):
+                mpiroot = ci_site_config.impi_root
+            else:
+                mpiroot = "{}/mpich".format(self.libfab_installpath)
+            if (self.util_prov):
+                prov = "\"{};{}\"".format(self.core_prov, self.util_prov)
+            else:
+                prov = self.core_prov
+
+            if (timeout != None):
+                os.environ['MPIEXEC_TIMEOUT']=timeout
+
+            opts = "-n {np} -hosts {s},{c} -mpi_root={mpiroot} \
+                    -libfabric_path={installpath}/lib -prov {provider} "  \
+                    .format(np=nprocs, s=self.server, c=self.client, \
+                            provider=prov, mpiroot=mpiroot, \
+                            installpath=self.libfab_installpath)
+
+        elif (self.mpi == "ompi"):
+            print(self.mpi)
+
+        return opts
+
+    @property
+    def execute_condn(self):
+        return True if (self.mpi == 'impi' and  self.core_prov != 'psm2') else False
+
+    def execute_cmd(self, testgroupname):
+        print("Running Tests: " + testgroupname)
+        tests = []
+        time = None
+        os.chdir("{}/{}".format(self.mpichsuitepath,testgroupname))
+        tests = self.testgroup(testgroupname)
+        for test in tests:
+            testname = test[0]
+            nprocs = test[1]
+            args = test[2:]
+            for item in args:
+               itemlist =  item.split('=')
+               if (itemlist[0] == 'timelimit'):
+                   time = itemlist[1]
+            opts = self.options(nprocs, timeout=time)
+            testcmd = self.cmd + opts +"./{}".format(testname)
+            outputcmd = shlex.split(testcmd)
+            common.run_command(outputcmd)
+        os.chdir(self.pwd)
+
 class MpiTestStress(MpiTests):
 
     def __init__(self, jobname, buildno, testname, core_prov, fabric,