Test scripts

tools/slurm-test.sh

@@ -1,120 +1,7 @@
 #!/bin/bash
-set -eux
 
-cd /data
-echo "#!/bin/bash
-#BB create_persistent name=mytestbuffer capacity=4000GB access=striped type=scratch" > create-persistent.sh
+source "$( dirname $0 )/tests/config"
 
-echo "#!/bin/bash
-#BB destroy_persistent name=mytestbuffer" > delete-persistent.sh
-
-echo "#!/bin/bash
-#DW jobdw capacity=2TB access_mode=striped,private type=scratch
-#DW stage_in source=/usr/local/bin/dacd destination=\$DW_JOB_STRIPED/filename1 type=file
-#DW stage_out source=\$DW_JOB_STRIPED/outdir destination=/tmp/perjob type=directory
-
-env
-df -h
-
-mkdir \$DW_JOB_STRIPED/outdir
-df -h > \$DW_JOB_STRIPED/outdir/dfoutput
-ls -al \$DW_JOB_STRIPED > \$DW_JOB_STRIPED/outdir/lsoutput
-file \$DW_JOB_STRIPED/filename1 > \$DW_JOB_STRIPED/outdir/stageinfile
-
-echo \$HOSTNAME
-" > use-perjob.sh
-
-echo "#!/bin/bash
-#DW persistentdw name=mytestbuffer
-#DW stage_in source=/usr/local/bin/dacd destination=\$DW_PERSISTENT_STRIPED_mytestbuffer/filename1 type=file
-#DW stage_out source=\$DW_PERSISTENT_STRIPED_mytestbuffer/outdir destination=/tmp/persistent type=directory
-
-env
-df -h
-
-mkdir -p \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir
-echo \$SLURM_JOBID >> \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir/jobids
-ls -al \$DW_PERSISTENT_STRIPED_mytestbuffer >> \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir/lsoutput
-file \$DW_PERSISTENT_STRIPED_mytestbuffer/filename1 >> \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir/stageinfile
-
-echo \$HOSTNAME
-" > use-persistent.sh
-
-echo "#!/bin/bash
-#DW jobdw capacity=2TB access_mode=striped,private type=scratch
-#DW persistentdw name=mytestbuffer
-#DW stage_in source=/usr/local/bin/dacd destination=\$DW_PERSISTENT_STRIPED_mytestbuffer/filename1 type=file
-#DW stage_out source=\$DW_PERSISTENT_STRIPED_mytestbuffer/outdir destination=/tmp/persistent type=directory
-
-env
-df -h
-
-touch \$DW_JOB_STRIPED/\$( date +%F )
-ls -al \$DW_JOB_STRIPED
-
-mkdir -p \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir
-echo \$SLURM_JOBID >> \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir/jobids
-ls -al \$DW_PERSISTENT_STRIPED_mytestbuffer >> \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir/lsoutput
-file \$DW_PERSISTENT_STRIPED_mytestbuffer/filename1 >> \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir/stageinfile
-
-echo \$HOSTNAME
-" > use-multiple.sh
-
-# Ensure Slurm is setup with the cluster name
-/usr/bin/sacctmgr --immediate add cluster name=linux || true
-
-SLEEP_INTERVAL=10
-
-echo "Wait for startup to complete..."
-sleep $SLEEP_INTERVAL
-
-scontrol show burstbuffer
-scontrol show bbstat
-
-echo "***Create persistent buffer***"
-cat create-persistent.sh
-su slurm -c 'sbatch create-persistent.sh'
-
-sleep $SLEEP_INTERVAL
-scontrol show burstbuffer
-squeue
-
-echo "***Create per job buffer***"
-su slurm -c 'srun --bb="capacity=100GB" bash -c "sleep 5 && echo \$HOSTNAME"'
-
-scontrol show burstbuffer
-squeue
-
-echo "***Use persistent buffer***"
-id centos &>/dev/null || adduser centos
-cat use-multiple.sh
-su centos -c 'sbatch --array=1-10 use-multiple.sh'
-su centos -c 'sbatch use-multiple.sh'
-su centos -c 'sbatch use-multiple.sh'
-su centos -c 'sbatch use-multiple.sh'
-su centos -c 'sbatch use-multiple.sh'
-cat use-persistent.sh
-su centos -c 'sbatch use-persistent.sh'
-cat use-perjob.sh
-su centos -c 'sbatch use-perjob.sh'
-
-squeue
-
-sleep $SLEEP_INTERVAL
-scontrol show burstbuffer
-squeue
-sleep $SLEEP_INTERVAL
-
-#echo "***Delete persistent buffer***"
-#cat delete-persistent.sh
-#su slurm -c 'sbatch delete-persistent.sh'
-#sleep $SLEEP_INTERVAL
-
-#echo "***Show all buffers are cleaned up***"
-scontrol show burstbuffer
-squeue
-sleep $SLEEP_INTERVAL
-scontrol show burstbuffer
-squeue
-scontrol show bbstat
-sacct -o jobid'%10',reqtres'%45',alloctres'%45' -X
+for i in tests/*.sh; do
+    timeout $timeout $i || echo test $i timed out
+done

tools/tests/config

@@ -0,0 +1,6 @@
+file=bufferfile.txt
+outfile=bufferout.txt
+timeout=1h
+jobtimeout=30m
+size=2TB
+nodes=2

tools/tests/functions

@@ -0,0 +1,14 @@
+wait_for_jobs() {
+  set +e
+  timeout $jobtimeout bash -c \
+  'until [ -z "$(squeue -h -n $name)" ]; do
+    sleep 10
+  done'
+  timeoutrc=$?
+  set -e
+  if [ ! $timeoutrc -eq 0 ]; then
+    echo timeout $jobtimeout reached waiting for jobs to complete:
+    squeue -n $name
+    return 1
+  fi
+}

tools/tests/over_allocation.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e
+
+export name=test$RANDOM
+scratch=$HOME/dac-tests-$name
+output=$scratch/slurm.out
+
+source "$( dirname $0 )/config"
+source "$( dirname $0 )/functions"
+
+# Calculate the size of the buffers to use, we'll create two each needing 2/3 of the space
+export $( scontrol show bu | head -n1 )
+size=$( expr ${TotalSpace//[!0-9]/} / 3 \* 2 )${TotalSpace//[0-9]/}
+
+create_files() {
+  mkdir -p $scratch
+  touch $scratch/$file
+
+  echo "#!/bin/bash
+#DW jobdw capacity=$size access_mode=striped type=scratch
+#DW stage_in source=$scratch/$file destination=\$DW_JOB_STRIPED/$file type=file
+#DW stage_out source=\$DW_JOB_STRIPED/$file destination=$scratch/$outfile type=file
+#SBATCH --output=$output --open-mode=append
+test -n \"\$DW_JOB_STRIPED\" || echo 'environment variable \$DW_JOB_STRIPED is not set'
+test -d \"\$DW_JOB_STRIPED\" || echo 'environment variable \$DW_JOB_STRIPED is not a directory'
+touch \"\$DW_JOB_STRIPED/$file\" || echo 'failed to touch file in \$DW_JOB_STRIPED'" > $scratch/over-allocate.sh
+}
+
+submit_jobs() {
+  for num in {1..2}; do
+    sbatch --job-name=$name --quiet $scratch/over-allocate.sh
+  done
+}
+
+cleanup() {
+  scancel -n $name -u $USER
+  rm -rf "$scratch"
+  wait_for_jobs
+}
+
+source "$( dirname $0 )/template"

tools/tests/perjob_buffer.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+set -e
+
+export name=test$RANDOM
+scratch=$HOME/dac-tests-$name
+output=$scratch/slurm.out
+
+source "$( dirname $0 )/config"
+source "$( dirname $0 )/functions"
+
+create_files() {
+  mkdir -p $scratch
+  touch $scratch/$file
+
+  echo "#!/bin/bash
+#DW jobdw capacity=$size access_mode=private type=scratch
+#SBATCH --output=$output --open-mode=append --nodes=$nodes
+set -e
+test -n \"\$DW_JOB_PRIVATE\" || echo 'environment variable \$DW_JOB_PRIVATE is not set'
+test -d \"\$DW_JOB_PRIVATE\" || echo 'environment variable \$DW_JOB_PRIVATE is not a directory'
+touch \"\$DW_JOB_PRIVATE/$file\" || echo 'failed to touch file in \$DW_JOB_PRIVATE'
+stat_uid=\$( stat -c %U \$DW_JOB_PRIVATE/ )
+stat_gid=\$( stat -c %G \$DW_JOB_PRIVATE/ )
+stat_perm=\$( stat -c %a \$DW_JOB_PRIVATE/ )
+if [[ \$stat_uid != $USER ]]; then
+  echo \"UID of \$DW_JOB_PRIVATE is \$stat_uid and not $USER\"; fi
+if [[ \$stat_gid != $USER ]]; then
+  echo \"GID of \$DW_JOB_PRIVATE is \$stat_gid and not $USER\"; fi
+if [[ \$stat_perm != 700 ]]; then
+  echo \"Permissions of \$DW_JOB_PRIVATE are \$stat_perm and not 700\"; fi" > $scratch/use-private.sh
+
+
+  echo "#!/bin/bash
+#DW jobdw capacity=$size access_mode=striped type=scratch
+#DW stage_in source=$scratch/$file destination=\$DW_JOB_STRIPED/$file type=file
+#DW stage_out source=\$DW_JOB_STRIPED/$file destination=$scratch/$outfile type=file
+#SBATCH --output=$output --open-mode=append --nodes=$nodes
+set -e
+test -n \"\$DW_JOB_STRIPED\" || echo 'environment variable \$DW_JOB_STRIPED is not set'
+test -d \"\$DW_JOB_STRIPED\" || echo 'environment variable \$DW_JOB_STRIPED is not a directory'
+test -f \"\$DW_JOB_STRIPED/$file\" || echo 'stage in file \$DW_JOB_STRIPED/$file does not exist'
+touch \"\$DW_JOB_STRIPED/$file\" || echo 'failed to touch file in \$DW_JOB_STRIPED'
+stat_uid=\$( stat -c %U \$DW_JOB_STRIPED/ )
+stat_gid=\$( stat -c %G \$DW_JOB_STRIPED/ )
+stat_perm=\$( stat -c %a \$DW_JOB_STRIPED/ )
+if [[ \$stat_uid != $USER ]]; then
+  echo \"UID of \$DW_JOB_STRIPED is \$stat_uid and not $USER\"; fi
+if [[ \$stat_gid != $USER ]]; then
+  echo \"GID of \$DW_JOB_STRIPED is \$stat_gid and not $USER\"; fi
+if [[ \$stat_perm != 700 ]]; then
+  echo \"Permissions of \$DW_JOB_STRIPED are \$stat_perm and not 700\"; fi" > $scratch/use-striped.sh
+
+  echo "#!/bin/bash
+#DW jobdw capacity=$size access_mode=private,striped type=scratch
+#DW stage_in source=$scratch/$file destination=\$DW_JOB_STRIPED/$file type=file
+#DW stage_out source=\$DW_JOB_STRIPED/$file destination=$scratch/$file type=file
+#SBATCH --output=$output --open-mode=append --nodes=$nodes
+set -e
+test -n \"\$DW_JOB_PRIVATE\" || echo 'environment variable \$DW_JOB_PRIVATE is not set'
+test -n \"\$DW_JOB_STRIPED\" || echo 'environment variable \$DW_JOB_STRIPED is not set'
+test -d \"\$DW_JOB_PRIVATE\" || echo 'environment variable \$DW_JOB_PRIVATE is not a directory'
+test -d \"\$DW_JOB_STRIPED\" || echo 'environment variable \$DW_JOB_STRIPED is not a directory'
+test -f \"\$DW_JOB_STRIPED/$file\" || echo 'stage in file \$DW_JOB_STRIPED/$file does not exist'
+touch \"\$DW_JOB_PRIVATE/$file\" || echo 'failed to touch file in \$DW_JOB_PRIVATE'
+touch \"\$DW_JOB_STRIPED/$file\" || echo 'failed to touch file in \$DW_JOB_STRIPED'
+stat_uid=\$( stat -c %U \$DW_JOB_STRIPED/ )
+stat_gid=\$( stat -c %G \$DW_JOB_STRIPED/ )
+stat_perm=\$( stat -c %a \$DW_JOB_STRIPED/ )
+if [[ \$stat_uid != $USER ]]; then
+  echo \"UID of \$DW_JOB_STRIPED is \$stat_uid and not $USER\"; fi
+if [[ \$stat_gid != $USER ]]; then
+  echo \"GID of \$DW_JOB_STRIPED is \$stat_gid and not $USER\"; fi
+if [[ \$stat_perm != 700 ]]; then
+  echo \"Permissions of \$DW_JOB_STRIPED are \$stat_perm and not 700\"; fi
+stat_uid=\$( stat -c %U \$DW_JOB_PRIVATE/ )
+stat_gid=\$( stat -c %G \$DW_JOB_PRIVATE/ )
+stat_perm=\$( stat -c %a \$DW_JOB_PRIVATE/ )
+if [[ \$stat_uid != $USER ]]; then
+  echo \"UID of \$DW_JOB_PRIVATE is \$stat_uid and not $USER\"; fi
+if [[ \$stat_gid != $USER ]]; then
+  echo \"GID of \$DW_JOB_PRIVATE is \$stat_gid and not $USER\"; fi
+if [[ \$stat_perm != 700 ]]; then
+  echo \"Permissions of \$DW_JOB_PRIVATE are \$stat_perm and not 700\"; fi" > $scratch/use-both.sh
+}
+
+submit_jobs() {
+  sbatch --job-name=$name --quiet $scratch/use-striped.sh
+  sbatch --job-name=$name --quiet $scratch/use-private.sh
+  sbatch --job-name=$name --quiet $scratch/use-both.sh
+  sbatch --job-name=$name --quiet --array=1-10 $scratch/use-striped.sh
+  sbatch --job-name=$name --quiet --array=1-10 $scratch/use-private.sh
+  sbatch --job-name=$name --quiet --array=1-10 $scratch/use-both.sh
+}
+
+cleanup() {
+  scancel -n $name -u $USER
+  rm -rf "$scratch"
+  wait_for_jobs
+}
+
+source "$( dirname $0 )/template"

tools/tests/persistent_buffer.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -e
+
+export name=test$RANDOM
+scratch=$HOME/dac-tests-$name
+output=$scratch/slurm.out
+
+source "$( dirname $0 )/config"
+source "$( dirname $0 )/functions"
+
+create_files() {
+  mkdir -p $scratch
+  touch $scratch/$file
+
+  echo "#!/bin/bash
+#BB create_persistent name=$name capacity=$size access=striped type=scratch
+#SBATCH --output=$output --open-mode=append
+" > $scratch/create-persistent.sh
+
+  echo "#!/bin/bash
+#DW persistentdw name=$name
+#DW stage_in source=$scratch/$file destination=\$DW_PERSISTENT_STRIPED_$name/$file type=file
+#DW stage_out source=\$DW_PERSISTENT_STRIPED_$name/$file destination=$scratch/$outfile type=file
+#SBATCH --output=$output --open-mode=append --nodes=$nodes
+test -n \"\$DW_PERSISTENT_STRIPED_$name\" || echo environment variable is not set
+test -d \"\$DW_PERSISTENT_STRIPED_$name\" || echo environment varibale is not a directory
+test -f \"\$DW_PERSISTENT_STRIPED_$name/$file\" || echo stage in file $DW_PERSISTENT_STRIPED_$name/$file does not exist
+echo \$SLURM_JOBID > \"\$DW_PERSISTENT_STRIPED_$name/$outfile\" || echo failed to write to file in buffer
+if [[ \$( stat -c %U \$DW_PERSISTENT_STRIPED_$name/ ) != $USER ]]; then
+  echo \"UID of \$DW_PERSISTENT_STRIPED_$name is not $USER\"; fi
+if [[ \$( stat -c %G \$DW_PERSISTENT_STRIPED_$name/ ) != root ]]; then
+  echo \"GID of \$DW_PERSISTENT_STRIPED_$name is not root\"; fi
+if [[ \$( stat -c %a \$DW_PERSISTENT_STRIPED_$name/ ) != 700 ]]; then
+  echo \"Permissions of \$DW_PERSISTENT_STRIPED_$name is not 700\"; fi" > $scratch/use-persistent.sh
+
+  echo "#!/bin/bash
+#BB destroy_persistent name=$name
+#SBATCH --output=$output --open-mode=append
+" > $scratch/delete-persistent.sh
+}
+
+submit_jobs() {
+  sbatch --job-name=$name --quiet $scratch/create-persistent.sh
+  sbatch --job-name=$name --quiet $scratch/use-persistent.sh
+}
+
+cleanup() {
+  scancel -n $name -u $USER
+  sbatch --job-name=$name --quiet $scratch/delete-persistent.sh
+  rm -rf "$scratch"
+  wait_for_jobs
+}
+
+source "$( dirname $0 )/template"

tools/tests/template

@@ -0,0 +1,25 @@
+trap cleanup EXIT
+
+create_files
+submit_jobs
+wait_for_jobs
+
+# Ignore slurmstepd errors
+if [ -f $output ]; then
+  sed '/rmdir(\/sys\/fs\/cgroup/d' -i $output
+  sed '/Cannot open \/proc\/fs\/lustre/d' -i $output
+fi
+
+# Any output means error
+if [ -s $output ]; then
+  cat $output
+  exit 1
+fi
+
+if [ ! -f $scratch/$outfile ]; then
+  echo Stage out file does not exist
+  exit 1
+elif [ -s $scratch/$outfile ]; then
+  echo Stage out file is empty
+  exit 1
+fi