From 4a1f319195866f5946c69aef3bbdcf79fc687a0b Mon Sep 17 00:00:00 2001 From: Nikhil Nanal Date: Mon, 18 May 2020 20:07:09 -0700 Subject: [PATCH] Jenkins-CI: enabled spawn tests build.py: added code to build mpich_test_suite. tests.py: added class for mpich_suite. runtests.py: added calls to run the spawn tests. Note: Currently not enabled for psm2 provider due to errors. Signed-off-by: Nikhil Nanal --- contrib/intel/jenkins/Jenkinsfile | 51 +++++++++++----------- contrib/intel/jenkins/build.py | 24 ++++++++++- contrib/intel/jenkins/run.py | 14 ++++++- contrib/intel/jenkins/runtests.py | 8 +++- contrib/intel/jenkins/tests.py | 70 ++++++++++++++++++++++++++++++- 5 files changed, 135 insertions(+), 32 deletions(-) diff --git a/contrib/intel/jenkins/Jenkinsfile b/contrib/intel/jenkins/Jenkinsfile index 1253bfc7306..fe992a2f418 100644 --- a/contrib/intel/jenkins/Jenkinsfile +++ b/contrib/intel/jenkins/Jenkinsfile @@ -44,48 +44,47 @@ pipeline { stage ('build-shmem') { steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { + withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']){ sh """ python3.7 contrib/intel/jenkins/build.py 'shmem' --ofi_build_mode='dbg' echo 'shmem benchmarks built successfully' """ - } } - } - - stage ('build OMPI_bm') { - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { - sh """ - python3.7 contrib/intel/jenkins/build.py 'ompi_benchmarks' --ofi_build_mode='dbg' - echo 'mpi benchmarks with ompi - built successfully' - """ - } + } + } + stage('build MPICH_bm') { + steps { + withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']){ + sh """ + python3.7 contrib/intel/jenkins/build.py 'mpich_benchmarks' --ofi_build_mode='dbg' + echo "mpi benchmarks with mpich - built successfully" + """ } - } - - stage('build IMPI_bm') { - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { + } + } + stage('build IMPI_bm') { + steps { + withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']){ sh """ python3.7 contrib/intel/jenkins/build.py 'impi_benchmarks' --ofi_build_mode='dbg' echo 'mpi benchmarks with impi - built successfully' """ + } } - } - } - - stage('build MPICH_bm') { - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { + } + + stage ('build OMPI_bm') { + steps { + withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']){ sh """ - python3.7 contrib/intel/jenkins/build.py 'mpich_benchmarks' --ofi_build_mode='dbg' - echo "mpi benchmarks with mpich - built successfully" + python3.7 contrib/intel/jenkins/build.py 'ompi_benchmarks' --ofi_build_mode='dbg' + echo 'mpi benchmarks with ompi - built successfully' """ } } } - stage('parallel-tests') { + + stage('parallel-tests') { parallel { stage('eth-tcp-dbg') { agent {node {label 'eth'}} diff --git a/contrib/intel/jenkins/build.py b/contrib/intel/jenkins/build.py index ed26374cbdc..e84b3a75213 100755 --- a/contrib/intel/jenkins/build.py +++ b/contrib/intel/jenkins/build.py @@ -10,6 +10,7 @@ import shlex import common import re +import shutil def build_libfabric(libfab_install_path, mode): @@ -138,13 +139,30 @@ def build_mpi(mpi, mpisrc, mpi_install_path, libfab_install_path, ofi_build_mod cmd.append("--enable-fortran=no") cmd.append("--with-device=ch4:ofi") cmd.append("--enable-ch4-direct=netmod") - configure_cmd = shlex.split(" ".join(cmd)) common.run_command(configure_cmd) common.run_command(["make", "clean"]) common.run_command(["make", "install", "-j32"]) +def build_mpich_suite(mpi, mpi_install_path, libfab_install_path): + + mpich_suite_path = '{}/test/'.format(ci_site_config.mpich_src) + mpichsuite_installpath= "{}/mpichsuite/test".format(mpi_install_path) + pwd = os.getcwd() + if (mpi == 'impi'): + os.chdir("{}/mpi".format(mpich_suite_path)) + cmd = ["./configure", "--with-mpi={}/intel64" \ + .format(ci_site_config.impi_root)] + + configure_cmd = shlex.split(" ".join(cmd)) + common.run_command(configure_cmd) + common.run_command(["make", "all","-j32"]) + shutil.copytree(mpich_suite_path, mpichsuite_installpath) + common.run_command(["make", "distclean"]) + os.chdir(pwd) + + def build_stress_bm(mpi, mpi_install_path, libfab_install_path): @@ -247,7 +265,9 @@ def build_osu_bm(mpi, mpi_install_path, libfab_install_path): else ci_site_config.ompi_src # only need to build ompi or mpich, impi is available as binary build_mpi(mpi, mpisrc, mpi_install_path, install_path, ofi_build_mode) - + + # build mpich_test_suite + build_mpich_suite(mpi, mpi_install_path, install_path) # run stress and osu benchmarks for all mpitypes build_stress_bm(mpi, mpi_install_path, install_path) build_osu_bm(mpi, mpi_install_path, install_path) diff --git a/contrib/intel/jenkins/run.py b/contrib/intel/jenkins/run.py index 683da131349..7873f0f1e77 100755 --- a/contrib/intel/jenkins/run.py +++ b/contrib/intel/jenkins/run.py @@ -75,7 +75,19 @@ def intel_mpi_benchmark(core, hosts, mpi, mode, util=None): print("skipping {} as execute condition fails"\ .format(imb_test.testname)) print("----------------------------------------------------------------------------------------\n") - + +#mpich_test_suite +def mpich_test_suite(core, hosts, mpi, mode, util=None): + mpich_tests = tests.MpichTestSuite(jobname=jbname,buildno=bno,\ + testname="MpichTestSuite",core_prov=core, fabric=fab,\ + mpitype=mpi, hosts=hosts, ofi_build_mode=mode, \ + util_prov=util) + if (mpich_tests.execute_condn == True and \ + mpich_tests.mpi_gen_execute_condn == True): + print("Running mpich test suite: Spawn coll, comm, dt Tests for {}-{}-{}-{}".format(core, util, fab, mpi)) + os.environ["MPITEST_RETURN_WITH_CODE"] = "1" + mpich_tests.execute_cmd("spawn") + #mpi_stress benchmark tests def mpistress_benchmark(core, hosts, mpi, mode, util=None): diff --git a/contrib/intel/jenkins/runtests.py b/contrib/intel/jenkins/runtests.py index a7a1b1b2f4d..c3cfe285d8f 100755 --- a/contrib/intel/jenkins/runtests.py +++ b/contrib/intel/jenkins/runtests.py @@ -50,6 +50,7 @@ run.fabtests(args_core, hosts, ofi_build_mode) run.shmemtest(args_core, hosts, ofi_build_mode) for mpi in mpilist: + run.mpich_test_suite(args_core, hosts, mpi, ofi_build_mode) run.intel_mpi_benchmark(args_core, hosts, mpi, ofi_build_mode) run.mpistress_benchmark(args_core, hosts, mpi, ofi_build_mode) run.osu_benchmark(args_core, hosts, mpi, ofi_build_mode) @@ -58,10 +59,13 @@ run.fabtests(args_core, hosts, ofi_build_mode, util=args_util) run.shmemtest(args_core, hosts, ofi_build_mode, util=args_util) for mpi in mpilist: + run.mpich_test_suite(args_core, hosts, mpi, ofi_build_mode, \ + util=args_util) + run.intel_mpi_benchmark(args_core, hosts, mpi, ofi_build_mode, \ - util=args_util,) + util=args_util) run.mpistress_benchmark(args_core, hosts, mpi, ofi_build_mode, \ - util=args_util) + util=args_util) run.osu_benchmark(args_core, hosts, mpi, ofi_build_mode, \ util=args_util) else: diff --git a/contrib/intel/jenkins/tests.py b/contrib/intel/jenkins/tests.py index ac9ecc03a66..e79c71901b0 100755 --- a/contrib/intel/jenkins/tests.py +++ b/contrib/intel/jenkins/tests.py @@ -207,7 +207,6 @@ def __init__(self, jobname, buildno, testname, core_prov, fabric, fabric, hosts, ofi_build_mode, util_prov) self.mpi = mpitype - @property def cmd(self): if (self.mpi == "impi" or self.mpi == "mpich"): @@ -358,7 +357,76 @@ def execute_cmd(self): outputcmd = shlex.split(command + self.rma.imb_cmd) common.run_command(outputcmd) +class MpichTestSuite(MpiTests): + + def __init__(self, jobname, buildno, testname, core_prov, fabric, + mpitype, hosts, ofi_build_mode, util_prov=None): + super().__init__(jobname, buildno, testname, core_prov, fabric, + mpitype, hosts, ofi_build_mode, util_prov) + self.mpichsuitepath = "{}/{}/mpichsuite/test/mpi/" \ + .format(self.libfab_installpath, self.mpi) + self.pwd = os.getcwd() + + def testgroup(self, testgroupname): + testpath = "{}/{}".format(self.mpichsuitepath, testgroupname) + tests = [] + with open("{}/testlist".format(testpath)) as file: + for line in file: + if(line[0] != '#' and line[0] != '\n'): + tests.append((line.rstrip('\n')).split(' ')) + + return tests + + def options(self, nprocs, timeout=None): + if (self.mpi == "impi" or self.mpi == "mpich"): + if (self.mpi == "impi"): + mpiroot = ci_site_config.impi_root + else: + mpiroot = "{}/mpich".format(self.libfab_installpath) + if (self.util_prov): + prov = "\"{};{}\"".format(self.core_prov, self.util_prov) + else: + prov = self.core_prov + + if (timeout != None): + os.environ['MPIEXEC_TIMEOUT']=timeout + + opts = "-n {np} -hosts {s},{c} -mpi_root={mpiroot} \ + -libfabric_path={installpath}/lib -prov {provider} " \ + .format(np=nprocs, s=self.server, c=self.client, \ + provider=prov, mpiroot=mpiroot, \ + installpath=self.libfab_installpath) + + elif (self.mpi == "ompi"): + print(self.mpi) + + return opts + + @property + def execute_condn(self): + return True if (self.mpi == 'impi' and self.core_prov != 'psm2') else False + + def execute_cmd(self, testgroupname): + print("Running Tests: " + testgroupname) + tests = [] + time = None + os.chdir("{}/{}".format(self.mpichsuitepath,testgroupname)) + tests = self.testgroup(testgroupname) + for test in tests: + testname = test[0] + nprocs = test[1] + args = test[2:] + for item in args: + itemlist = item.split('=') + if (itemlist[0] == 'timelimit'): + time = itemlist[1] + opts = self.options(nprocs, timeout=time) + testcmd = self.cmd + opts +"./{}".format(testname) + outputcmd = shlex.split(testcmd) + common.run_command(outputcmd) + os.chdir(self.pwd) + class MpiTestStress(MpiTests): def __init__(self, jobname, buildno, testname, core_prov, fabric,