diff --git a/janis_assistant/containers/base.py b/janis_assistant/containers/base.py index fbd197a4..65bced39 100644 --- a/janis_assistant/containers/base.py +++ b/janis_assistant/containers/base.py @@ -1,6 +1,6 @@ from abc import abstractmethod, ABC from enum import Enum -from typing import Dict +from typing import Dict, Optional class ContainerType(Enum): @@ -41,7 +41,7 @@ def get_container_type(): @staticmethod @abstractmethod - def test_available_by_getting_version() -> str: + def test_available_by_getting_version(command: Optional[str] = None) -> str: """ Test if the container environment is available, raise the ContainerEnvNotFound if not. :return: diff --git a/janis_assistant/containers/docker.py b/janis_assistant/containers/docker.py index be8a1474..b81e3c6a 100644 --- a/janis_assistant/containers/docker.py +++ b/janis_assistant/containers/docker.py @@ -1,5 +1,5 @@ import subprocess -from typing import Dict +from typing import Dict, Optional from janis_core import Logger @@ -31,7 +31,7 @@ def get_container_type(): return ContainerType.docker @staticmethod - def test_available_by_getting_version() -> str: + def test_available_by_getting_version(command: Optional[str] = None) -> str: try: return subprocess.check_output(["docker", "-v"]).decode() except subprocess.CalledProcessError as e: diff --git a/janis_assistant/containers/singularity.py b/janis_assistant/containers/singularity.py index edd170b1..0ee98bfa 100644 --- a/janis_assistant/containers/singularity.py +++ b/janis_assistant/containers/singularity.py @@ -1,7 +1,7 @@ import os import subprocess -from typing import Dict +from typing import Dict, Optional from janis_core import Logger @@ -41,7 +41,7 @@ def get_build_instructions_for(containerlocation: str, docker: str): return ["singularity", "pull", containerlocation, "docker://" + docker] @staticmethod - def test_available_by_getting_version() -> str: + def test_available_by_getting_version(command: Optional[str] = None) -> str: try: version = subprocess.check_output(["singularity", "--version"]).decode() import re @@ -58,8 +58,18 @@ def test_available_by_getting_version() -> str: ) return version - except subprocess.CalledProcessError as e: - raise Container.ContainerEnvNotFound("singularity", e) + # except subprocess.CalledProcessError as e: + # raise Container.ContainerEnvNotFound("singularity", e) + except Exception as e: + + if command is not None: + try: + Logger.info("Trying to load singularity") + subprocess.run(command, shell=True) + except Exception as e: + raise Container.ContainerEnvNotFound("singularity", e) + else: + raise Container.ContainerEnvNotFound("singularity", e) def start_container(self): diff --git a/janis_assistant/main.py b/janis_assistant/main.py index e8db2b7a..77af442e 100644 --- a/janis_assistant/main.py +++ b/janis_assistant/main.py @@ -9,6 +9,7 @@ import os import sys import time +import subprocess from datetime import datetime from inspect import isclass from textwrap import dedent diff --git a/janis_assistant/management/workflowmanager.py b/janis_assistant/management/workflowmanager.py index b40e186e..fa3a55e9 100644 --- a/janis_assistant/management/workflowmanager.py +++ b/janis_assistant/management/workflowmanager.py @@ -288,8 +288,9 @@ def start_or_submit(self, run_in_background, watch=False): jc = metadb.prepared_job PreparedJob._instance = jc + metadb.containertype = jc._container.__name__ - metadb.containerversion = jc._container.test_available_by_getting_version() + metadb.containerversion = jc._container.test_available_by_getting_version(jc.template.template.setup_container_command()) self.database.submission_metadata.save_changes() @@ -310,6 +311,10 @@ def start_or_submit(self, run_in_background, watch=False): "--foreground", self.execution_dir, ] + + if jc.template.template.prejanis_hook() is not None: + command = [f"{jc.template.template.prejanis_hook()};"] + command + scriptdir = self.get_path_for_component(self.WorkflowManagerPath.configuration) logdir = self.get_path_for_component(self.WorkflowManagerPath.logs) diff --git a/janis_assistant/modifiers/base.py b/janis_assistant/modifiers/base.py index e78445b4..aa9c9d6a 100644 --- a/janis_assistant/modifiers/base.py +++ b/janis_assistant/modifiers/base.py @@ -73,7 +73,12 @@ def localise_inputs( if localise_secondary_files: try: - for sec in inptype.secondary_files() or []: + # Handle normal input type or array input type + secondary_files = inptype.secondary_files() + if inptype.is_array(): + secondary_files = inptype.subtype().secondary_files() + + for sec in secondary_files or []: sec_source = apply_secondary_file_format_to_filename(source, sec) out_sec_path = apply_secondary_file_format_to_filename( out_path, sec diff --git a/janis_assistant/modifiers/remotefilemodifier.py b/janis_assistant/modifiers/remotefilemodifier.py index 71ac4a16..61ac7e79 100644 --- a/janis_assistant/modifiers/remotefilemodifier.py +++ b/janis_assistant/modifiers/remotefilemodifier.py @@ -37,31 +37,26 @@ def inputs_modifier(self, tool: Tool, inputs: Dict, hints: Dict[str, str]) -> Di :return: modified input :rtype: dict """ - if not isinstance(tool, WorkflowBase): - return inputs - - wf: WorkflowBase = tool new_inputs = {} - for inpnode in wf.input_nodes.values(): + for inp in tool.tool_inputs(): modification_required = False - if isinstance(inpnode.datatype, File) or ( - isinstance(inpnode.datatype, Array) - and isinstance(inpnode.datatype.fundamental_type(), File) + if isinstance(inp.intype, File) or ( + isinstance(inp.intype, Array) + and isinstance(inp.intype.fundamental_type(), File) ): - if inpnode.id() in inputs and inputs[inpnode.id()] is not None: + if inp.id() in inputs and inputs[inp.id()] is not None: modification_required = True if modification_required: - doc: InputDocumentation = inpnode.doc - source = inputs[inpnode.id()] + source = inputs[inp.id()] basedir = self.cache_dir os.makedirs(basedir, exist_ok=True) - new_inputs[inpnode.id()] = self.localise_inputs( - inpnode.id(), - inpnode.datatype, + new_inputs[inp.id()] = self.localise_inputs( + inp.id(), + inp.intype, basedir, source, # mfranklin 2021-01-08: diff --git a/janis_assistant/templates/base.py b/janis_assistant/templates/base.py index fe01efed..542bca74 100644 --- a/janis_assistant/templates/base.py +++ b/janis_assistant/templates/base.py @@ -1,4 +1,5 @@ import sys +import os from abc import ABC, abstractmethod from datetime import datetime from typing import Type, Optional, List, Dict @@ -223,6 +224,9 @@ def prepare_run_test_command(self, test_command: List[str]) -> Optional[List]: """ return test_command + def setup_container_command(self): + return None + class SingularityEnvironmentTemplate(EnvironmentTemplate, ABC): def __init__( diff --git a/janis_assistant/templates/slurm.py b/janis_assistant/templates/slurm.py index c1a0e27f..1cac637f 100644 --- a/janis_assistant/templates/slurm.py +++ b/janis_assistant/templates/slurm.py @@ -225,3 +225,6 @@ def submit_detatched_resume( logsdir=logsdir, **kwargs, ) + + def setup_container_command(self): + return self.singularity_load_instructions