Merge branch 'main' into snake_config

bin/check_disk

@@ -4,3 +4,7 @@ ssh cl5n006 "( df | grep '/data1' | awk -F '[[:space:]]+' '{print \$5}' | sed 's
 ssh cl5n007 "( df | grep '/data1' | awk -F '[[:space:]]+' '{print \$5}' | sed 's/%//' ) > /work/microbiome/cl5n007_disk_usage"
 ssh cl5n008 "( df | grep '/data1' | awk -F '[[:space:]]+' '{print \$5}' | sed 's/%//' ) > /work/microbiome/cl5n008_disk_usage"
 ssh cl5n009 "( df | grep '/data1' | awk -F '[[:space:]]+' '{print \$5}' | sed 's/%//' ) > /work/microbiome/cl5n009_disk_usage"
+ssh cl5n010 "( df | grep '/data1' | awk -F '[[:space:]]+' '{print \$5}' | sed 's/%//' ) > /work/microbiome/cl5n010_disk_usage"
+ssh cl5n011 "( df | grep '/data1' | awk -F '[[:space:]]+' '{print \$5}' | sed 's/%//' ) > /work/microbiome/cl5n011_disk_usage"
+ssh cl5n012 "( df | grep '/data1' | awk -F '[[:space:]]+' '{print \$5}' | sed 's/%//' ) > /work/microbiome/cl5n012_disk_usage"
+ssh cl5n013 "( df | grep '/data1' | awk -F '[[:space:]]+' '{print \$5}' | sed 's/%//' ) > /work/microbiome/cl5n013_disk_usage"

bin/crontab

@@ -1 +1 @@
-*/10 * * * * bash /work/microbiome/sw/hpc_scripts/bin/check_disk
+*/10 * * * * bash /work/microbiome/sw/hpc_scripts/bin/check_disk >/dev/null 2>&1

bin/mpermissions

@@ -93,13 +93,13 @@ if __name__ == '__main__':
     folder2 = "'{}'".format(folder)
 
     dir_group_permissions = 'rwx'
-    file_group_permissions = 'rw-'
+    file_group_permissions = 'rwx'
     if args.no_group_write:
         dir_group_permissions = 'r-x'
-        file_group_permissions = 'r--'
+        file_group_permissions = 'r-x'
     cmd1 = 'find {} -type d -exec setfacl -dm g:microbiome-admin:rwx,g::{},u::rwx,o::--- {{}} +'.format(folder2, dir_group_permissions)
     cmd1b = 'find {} -type d -exec setfacl -m g:microbiome-admin:rwx,g::{},u::rwx,o::--- {{}} +'.format(folder2, dir_group_permissions)
-    cmd2 = 'find {} -type f -exec setfacl -m g:microbiome-admin:rw-,g::{},u::rw-,o::--- {{}} +'.format(folder2, file_group_permissions)
+    cmd2 = 'find {} -type f -exec setfacl -m g:microbiome-admin:rwx,g::{},u::rwx,o::--- {{}} +'.format(folder2, file_group_permissions)
     cmd3 = 'chgrp -R {} {}'.format(owner_group, folder2)
     cmd4 = 'find {} -type d -exec chmod g+s {{}} +'.format(folder2)
 

bin/mqsub

@@ -172,10 +172,8 @@ class script_format:
         try:
             current_conda_env = os.environ['CONDA_PREFIX']
         except:
-            current_conda_env = str('/mnt/hpccs01/work/microbiome/conda/bin/conda')
+            current_conda_env = None
         if current_conda_env:
-#            print('. /pkg/suse12/software/miniconda3/4.5.12/etc/profile.d/conda.sh',file=outfile)
-#            print('conda activate',file=outfile)
             print("conda activate '{}'".format(current_conda_env),file=outfile)  
         if args.command_file and (args.chunk_num or args.chunk_size):
             print("\nNUM_FAILED=0\n", file=outfile)
@@ -313,7 +311,8 @@ def setup_segregated_logs_directory(command_name):
         num_logs_dirs = 0
     segregated_logs_dir = os.path.join(logs_dir1, command_name+'-'+str(num_logs_dirs + 1))
     logging.info("Creating segregated log directory {}".format(segregated_logs_dir))
-    os.makedirs(segregated_logs_dir)
+    # Add exist_ok due to race condition with parallel mqsubs trying to create the same folder
+    os.makedirs(segregated_logs_dir, exist_ok=True)
 
     return segregated_logs_dir
 

bin/snakemake_mqsub

@@ -79,8 +79,12 @@ if __name__ == '__main__':
     job_properties = read_job_properties(jobscript)
 
     threads = job_properties['threads']
-    mem_mb = job_properties['resources']['mem_mb']
-    mem_gb = int(mem_mb / 1000)
+    if 'mem_mb' in job_properties['resources']:
+        mem_mb = job_properties['resources']['mem_mb']
+        mem_gb = int(mem_mb / 1000)
+        mem_argument = "-m {}".format(mem_gb)
+    else:
+        mem_argument = ''
 
     if 'runtime' in job_properties['resources']:
         runtime_mins = job_properties['resources']['runtime'] # Fails with snakemake == 7.16.0, but works with 7.30.1
@@ -109,9 +113,9 @@ if __name__ == '__main__':
     # But currently mqsub ignores this anyway, so definiting --name has no effect.
     job_name = os.path.basename(jobscript).replace('snakemake','')
 
-    cmd = "mqsub --no-email --quiet --bg --name {job_name} -t {threads} -m {mem} {hours} --script {script} {queue} {segregated_log_files_arg} {extra_mqsub_args} 2>&1".format(
+    cmd = "mqsub --no-email --quiet --bg --name {job_name} -t {threads} {mem} {hours} --script {script} {queue} {segregated_log_files_arg} {extra_mqsub_args} 2>&1".format(
         job_name=job_name, extra_mqsub_args=extra_mqsub_args,
-        threads=threads, script=jobscript, mem=mem_gb, hours=runtime_hours, queue=queue, 
+        threads=threads, script=jobscript, mem=mem_argument, hours=runtime_hours, queue=queue, 
         segregated_log_files_arg=segregated_log_files_arg)
 
     if args.dry_run:

cmr_bashrc_extras.bash

@@ -22,7 +22,8 @@ PROMPT_COMMAND="${PROMPT_COMMAND:+$PROMPT_COMMAND ; }"'echo $$ $USER \
                "$(history 1)" >> ~/.bash_eternal_history'
 
 #Path to kingfisher # symlinked /lustre/work-lustre/microbiome/sw/kingfisher-download/bin/kingfisher in $CONDA_PREFIX/envs/kingfisher/bin instead
-#export PATH=/lustre/work-lustre/microbiome/sw/kingfisher-download/bin:$PATH
+export PATH=/work/microbiome/sw/kingfisher_repos/kingfisher-v0.4.1/kingfisher-download/bin:$PATH
+export PATH=/work/microbiome/sw/recurm_repos/recurm/bin:$PATH
 
 #add RAM usage limits
 if [ `hostname` = "cl5n006" ] || [ `hostname` = "cl5n007" ] || [ `hostname` = "cl5n008" ] || [ `hostname` = "cl5n009" ]
@@ -41,7 +42,12 @@ case $- in *i*) echo "** cl5n006 disk usage (%): `cat /work/microbiome/cl5n006_d
 case $- in *i*) echo "** cl5n007 disk usage (%): `cat /work/microbiome/cl5n007_disk_usage` **"; esac
 case $- in *i*) echo "** cl5n008 disk usage (%): `cat /work/microbiome/cl5n008_disk_usage` **"; esac
 case $- in *i*) echo "** cl5n009 disk usage (%): `cat /work/microbiome/cl5n009_disk_usage` **"; esac
+case $- in *i*) echo "** cl5n010 disk usage (%): `cat /work/microbiome/cl5n010_disk_usage` **"; esac
+case $- in *i*) echo "** cl5n011 disk usage (%): `cat /work/microbiome/cl5n011_disk_usage` **"; esac
+case $- in *i*) echo "** cl5n012 disk usage (%): `cat /work/microbiome/cl5n012_disk_usage` **"; esac
+case $- in *i*) echo "** cl5n013 disk usage (%): `cat /work/microbiome/cl5n013_disk_usage` **"; esac
+
 
 # Setup snakemake config directories so it interfaces well with the PBS system
 mkdir -p ~/.config/snakemake
-cd ~/.config/snakemake && ln -sf /work/microbiome/sw/hpc_scripts/snakemake_configs/* . && cd ~
+cd ~/.config/snakemake && ln -sf /work/microbiome/sw/hpc_scripts/snakemake_configs/* . && cd $OLDPWD

snakemake_configs/mqsub-lyra/config.v8+.yaml

@@ -0,0 +1,8 @@
+executor: cluster-generic
+cluster-generic-submit-cmd: snakemake_mqsub --segregated-log-files --queue lyra
+cluster-generic-status-cmd: snakemake_mqstat
+jobs: 10000
+cluster-generic-cancel-cmd: qdel
+use-conda: true
+conda-frontend: mamba
+rerun-incomplete: true # Without this, snakemake will attempt to resume when rerunning a rule, which fails immediately without error

snakemake_configs/mqsub-lyra/config.yaml

@@ -4,4 +4,4 @@ jobs: 10000
 cluster-cancel: qdel
 use-conda: true
 conda-frontend: mamba
-rerun-incomplete: true
+rerun-incomplete: true # Without this, snakemake will attempt to resume when rerunning a rule, which fails immediately without error

snakemake_configs/mqsub/config.v8+.yaml

@@ -0,0 +1,8 @@
+executor: cluster-generic
+cluster-generic-submit-cmd: snakemake_mqsub --segregated-log-files
+cluster-generic-status-cmd: snakemake_mqstat
+jobs: 10000
+cluster-generic-cancel-cmd: qdel
+use-conda: true
+conda-frontend: mamba
+rerun-incomplete: true # Without this, snakemake will attempt to resume when rerunning a rule, which fails immediately without error

snakemake_configs/mqsub/config.yaml

@@ -4,4 +4,4 @@ jobs: 10000
 cluster-cancel: qdel
 use-conda: true
 conda-frontend: mamba
-rerun-incomplete: true
+rerun-incomplete: true # Without this, snakemake will attempt to resume when rerunning a rule, which fails immediately without error

windows_microbiome_ssh_config

@@ -2,21 +2,35 @@ Host *
     ServerAliveInterval 300
     ServerAliveCountMax 2
 
-Host transfer1
-    HostName eresdt001
-    User woodcrob
-    Compression yes
-    ProxyCommand ssh.exe -q -W %h:%p lyra
+Host lyra
+     HostName lyra-mb.qut.edu.au
+     User woodcrob
+     Compression yes
 
-Host transfer2
-    HostName eresdt002
-    User woodcrob
-    Compression yes
-    ProxyCommand ssh.exe -q -W %h:%p lyra
+Host lyra-mb
+     HostName lyra-mb.qut.edu.au
+     User woodcrob
+     Compression yes
 
-Host lyra
-    HostName lyra.qut.edu.au
-    User woodcrob
+Host lyra0
+     HostName lyra.qut.edu.au
+     User woodcrob
+     Compression yes
+
+Host lyra1
+     HostName lyra01.qut.edu.au
+     User woodcrob
+     Compression yes
+
+Host lyra3
+     HostName lyra03.qut.edu.au
+     User woodcrob
+     Compression yes
+
+Host lyra4
+     HostName lyra04.qut.edu.au
+     User woodcrob
+     Compression yes
 
 Host n5
     HostName cl5n005
@@ -61,4 +75,16 @@ Host n12
 Host n13
     HostName cl5n013
     User woodcrob
+    ProxyCommand ssh.exe -q -W %h:%p lyra
+
+Host transfer1
+    HostName eresdt001
+    User woodcrob
+    Compression yes
+    ProxyCommand ssh.exe -q -W %h:%p lyra
+
+Host transfer2
+    HostName eresdt002
+    User woodcrob
+    Compression yes
     ProxyCommand ssh.exe -q -W %h:%p lyra