Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[develop] upgrade slurm #5695

Merged
merged 1 commit into from
Oct 6, 2023
Merged

[develop] upgrade slurm #5695

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 4 additions & 10 deletions tests/integration-tests/tests/schedulers/test_slurm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ def test_slurm(
remote_command_executor = RemoteCommandExecutor(cluster, use_login_node=use_login_node)
slurm_root_path = _retrieve_slurm_root_path(remote_command_executor)
slurm_commands = scheduler_commands_factory(remote_command_executor)
_test_slurm_version(remote_command_executor)

if supports_impi:
_test_mpi_job_termination(remote_command_executor, test_datadir, slurm_commands, region, cluster)
Expand Down Expand Up @@ -267,18 +266,19 @@ def test_slurm_pmix(pcluster_config_reader, scheduler, clusters_factory, use_log
remote_command_executor = RemoteCommandExecutor(cluster, use_login_node=use_login_node)

# Ensure the expected PMIx version is listed when running `srun --mpi=list`.
# Since we're installing PMIx v3.1.5, we expect to see pmix and pmix_v3 in the output.
# Sample output:
# [ec2-user@ip-172-31-33-187 ~]$ srun 2>&1 --mpi=list
# srun: MPI types are...
# srun: none
# srun: openmpi
# srun: pmi2
# srun: pmix
# srun: pmix_v3
# srun: pmix_vX
#
# _vX is the Major number of the PMIx version installed and used to compile slurm.
# We check this in the cookbook, so we do not repeat the check here
mpi_list_output = remote_command_executor.run_remote_command("srun 2>&1 --mpi=list").stdout
assert_that(mpi_list_output).matches(r"\s+pmix($|\s+)")
Comment on lines 268 to 281
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would say that these are also canary-like tests and we don't have to test them here. But that's probably something for the future.

assert_that(mpi_list_output).matches(r"\s+pmix_v3($|\s+)")

# Compile and run an MPI program interactively
mpi_module = "openmpi"
Expand Down Expand Up @@ -1746,12 +1746,6 @@ def _gpu_resource_check(slurm_commands, partition, instance_type, instance_type_
assert_that(job_info).contains(f"TresPerNode=gres:gpu:{gpus_per_instance}", f"CpusPerTres=gres:gpu:{cpus_per_gpu}")


def _test_slurm_version(remote_command_executor):
logging.info("Testing Slurm Version")
version = remote_command_executor.run_remote_command("sinfo -V").stdout
assert_that(version).is_equal_to("slurm 23.02.4")


def _test_job_dependencies(slurm_commands, region, stack_name, scaledown_idletime):
logging.info("Testing cluster doesn't scale when job dependencies are not satisfied")
job_id = slurm_commands.submit_command_and_assert_job_accepted(
Expand Down
Loading