Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Addedd support for executing on slurm #30

Merged
merged 10 commits into from
Oct 16, 2023
Merged
4 changes: 3 additions & 1 deletion jarvis_util/shell/exec.py
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
This module provides mechanisms to execute binaries either locally or
remotely.
"""

from jarvis_util.shell.slurm_exec import SlurmExec
from .local_exec import LocalExec
from .pssh_exec import PsshExec
from .pssh_exec import SshExec
@@ -35,6 +35,8 @@ def __init__(self, cmd, exec_info=None):
self.exec_ = PsshExec(cmd, exec_info)
elif exec_type == ExecType.MPI:
exec_type = MpiVersion(exec_info).version
elif exec_type == ExecType.SLURM:
exec_type = SlurmExec(cmd, exec_info)

if exec_type == ExecType.MPICH:
self.exec_ = MpichExec(cmd, exec_info)
2 changes: 1 addition & 1 deletion jarvis_util/shell/exec_info.py
Original file line number Diff line number Diff line change
@@ -37,7 +37,7 @@ def __init__(self, exec_type=ExecType.LOCAL, nprocs=None, ppn=None,
hostfile=None, hosts=None, env=None,
sleep_ms=0, sudo=False, sudoenv=True, cwd=None,
collect_output=None, pipe_stdout=None, pipe_stderr=None,
hide_output=None, exec_async=False, stdin=None):
hide_output=None, exec_async=False, stdin=None, **kwargs):
"""

:param exec_type: How to execute a program. SSH, MPI, Local, etc.
14 changes: 6 additions & 8 deletions jarvis_util/shell/slurm_exec.py
Original file line number Diff line number Diff line change
@@ -19,23 +19,21 @@ class SlurmExec(LocalExec):
def __init__(self, cmd, exec_info):
"""
Execute a command through sbatch

:param cmd: A command (string) to execute
:param exec_info: Information needed by sbatch
"""

self.cmd = cmd
self.job_name = exec_info.job_name
self.num_nodes = exec_info.num_noodes
self.num_nodes = exec_info.num_nodes
self.ppn = exec_info.ppn
self.cpus_per_task = exec_info.cpus_per_task
self.time = exec_info.time
self.partition = exec_info.partition
self.mail_type = exec_info.mail_type
self.output = exec_info.pipe_stdout
self.error = exec_info.pipe_stderr
self.memory = exec_info.memory
self.gres = exec_info.gres
self.mem = exec_info.mem
self.gres = exec_info.gres
self.exclusive = exec_info.exclusive

super().__init__(self.slurmcmd(),
@@ -55,7 +53,7 @@ def generate_sbatch_command(self):
'mail_type': 'mail-type',
'output': 'output',
'error': 'error',
'memory': 'mem',
'mem': 'mem',
'gres': 'gres',
'exclusive': 'exclusive'
}
@@ -86,10 +84,10 @@ def __init__(self, job_name=None, num_nodes=1, **kwargs):
'mail_user', 'mem', 'gres', 'exclusive']
self.keys += allowed_options
# We use ppn, and the output and error file from the base Exec Info
self.job_name = job_name
self.num_nodes = num_nodes
for key in allowed_options:
if key in kwargs:
setattr(self, key, kwargs[key])
else:
setattr(self, key, None)
self.job_name = job_name
self.num_nodes = num_nodes
2 changes: 0 additions & 2 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
pyyaml
pylint==2.15.0
coverage==5.5
# coverage
coverage-lcov==0.2.4
# coverage-lcov
pytest==6.2.5
tabulate
18 changes: 13 additions & 5 deletions setup.py
Original file line number Diff line number Diff line change
@@ -2,14 +2,23 @@

setuptools.setup(
name="jarvis_util",
packages=setuptools.find_packages(),
scripts=['bin/pylsblk'],
version="0.0.1",
description='Utilities to aid with creating shell scripts within Python.',
long_description=open('README.md').read(),
long_description_content_type='text/markdown',
author="Luke Logan",
author_email="llogan@hawk.iit.edu",
description="Various wrappers around shell utilities",
url="https://github.com/scs-lab/jarvis-util",
classifiers = [
packages=setuptools.find_packages(),
install_requires=[
'pyyaml',
'pylint==2.15.0',
'coverage==5.5',
'coverage-lcov==0.2.4',
'pytest==6.2.5',
'tabulate'
],
classifiers=[
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Development Status :: 0 - Pre-Alpha",
@@ -20,5 +29,4 @@
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: Application Configuration",
],
long_description=""
)
Loading