From 55f2204ed866d16314c21194663d2cc602e97c3a Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Thu, 27 Feb 2020 10:31:06 +0100
Subject: [PATCH 01/14] Start building nf-core modules commands

---
 nf_core/modules.py | 20 ++++++++++++++++++++
 scripts/nf-core    | 28 +++++++++++++++++++++++++++-
 2 files changed, 47 insertions(+), 1 deletion(-)
 create mode 100644 nf_core/modules.py

diff --git a/nf_core/modules.py b/nf_core/modules.py
new file mode 100644
index 0000000000..d68f6fa414
--- /dev/null
+++ b/nf_core/modules.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+""" Code to handle DSL2 module imports from nf-core/modules
+"""
+
+from __future__ import print_function
+
+import os
+import requests
+import sys
+import tempfile
+
+def get_modules_filetree():
+    """
+    Fetch the file list from nf-core/modules
+    """
+    r = requests.get("https://api.github.com/repos/nf-core/modules/git/trees/master?recursive=1")
+    if r.status_code == 200:
+        print('Success!')
+    elif r.status_code == 404:
+        print('Not Found.')
diff --git a/scripts/nf-core b/scripts/nf-core
index 65e0311114..ee8017c1e5 100755
--- a/scripts/nf-core
+++ b/scripts/nf-core
@@ -16,6 +16,7 @@ import nf_core.launch
 import nf_core.licences
 import nf_core.lint
 import nf_core.list
+import nf_core.modules
 import nf_core.sync
 
 import logging
@@ -276,7 +277,7 @@ def bump_version(pipeline_dir, new_version, nextflow):
         nf_core.bump_version.bump_nextflow_version(lint_obj, new_version)
 
 
-@nf_core_cli.command('sync', help_priority=8)
+@nf_core_cli.command(help_priority=8)
 @click.argument(
     'pipeline_dir',
     type = click.Path(exists=True),
@@ -343,6 +344,31 @@ def sync(pipeline_dir, make_template_branch, from_branch, pull_request, username
             logging.error(e)
             sys.exit(1)
 
+## nf-core module subcommands
+@nf_core_cli.group(cls=CustomHelpOrder)
+def module():
+    """ Manage DSL 2 module imports """
+    pass
+
+@module.command(help_priority=1)
+def install():
+    """ Install a DSL2 module """
+    pass
+
+@module.command(help_priority=2)
+def remove():
+    """ Remove a DSL2 module """
+    pass
+
+@module.command(help_priority=3)
+def check():
+    """ Check that imported module code has not been modified """
+    pass
+
+@module.command(help_priority=4)
+def fix():
+    """ Replace imported module code with a freshly downloaded copy """
+    pass
 
 if __name__ == '__main__':
     click.echo(click.style("\n                                          ,--.", fg='green')+click.style("/",fg='black')+click.style(",-.", fg='green'), err=True)

From 4dd3314a6153fd8ae8b51a1b3aaa221bc34b3d59 Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Tue, 3 Mar 2020 18:41:48 +0100
Subject: [PATCH 02/14] Extend nf-core modules code: download files

Made proper start on nf-core modules install. Now gets details from GitHub and downloads tool files.
---
 nf_core/modules.py | 151 ++++++++++++++++++++++++++++++++++++++++++---
 scripts/nf-core    |  55 +++++++++++++----
 2 files changed, 185 insertions(+), 21 deletions(-)

diff --git a/nf_core/modules.py b/nf_core/modules.py
index d68f6fa414..398066f904 100644
--- a/nf_core/modules.py
+++ b/nf_core/modules.py
@@ -1,20 +1,151 @@
 #!/usr/bin/env python
-""" Code to handle DSL2 module imports from nf-core/modules
+"""
+Code to handle DSL2 module imports from nf-core/modules
 """
 
 from __future__ import print_function
 
+import base64
+import logging
 import os
 import requests
 import sys
 import tempfile
 
-def get_modules_filetree():
-    """
-    Fetch the file list from nf-core/modules
-    """
-    r = requests.get("https://api.github.com/repos/nf-core/modules/git/trees/master?recursive=1")
-    if r.status_code == 200:
-        print('Success!')
-    elif r.status_code == 404:
-        print('Not Found.')
+
+class PipelineModules(object):
+
+    def __init__(self):
+        """
+        Initialise the PipelineModules object
+        """
+        self.pipeline_dir = os.getcwd()
+        self.modules_file_tree = {}
+        self.modules_current_hash = None
+        self.modules_avail_tool_names = []
+
+
+    def list_modules(self):
+        """
+        Get available tool names from GitHub tree for nf-core/modules
+        and print as list to stdout
+        """
+        mods = PipelineModules()
+        mods.get_modules_file_tree()
+        logging.info("Tools available from nf-core/modules:\n")
+        # Print results to stdout
+        print("\n".join(mods.modules_avail_tool_names))
+
+    def install(self, tool):
+        mods = PipelineModules()
+        mods.get_modules_file_tree()
+
+        # Check that the supplied name is an available tool
+        if tool not in mods.modules_avail_tool_names:
+            logging.error("Tool '{}' not found in list of available modules.".format(tool))
+            logging.info("Use the command 'nf-core modules list' to view available tools")
+            return
+        logging.debug("Installing tool '{}' at modules hash {}".format(tool, mods.modules_current_hash))
+
+        # Check that we don't already have a folder for this tool
+        tool_dir = os.path.join(self.pipeline_dir, 'modules', 'tools', tool)
+        if(os.path.exists(tool_dir)):
+            logging.error("Tool directory already exists: {}".format(tool_dir))
+            logging.info("To update an existing tool, use the commands 'nf-core update' or 'nf-core fix'")
+            return
+
+        # Download tool files
+        files = mods.get_tool_file_urls(tool)
+        logging.debug("Fetching tool files:\n - {}".format("\n - ".join(files.keys())))
+        for filename, api_url in files.items():
+            dl_filename = os.path.join(self.pipeline_dir, 'modules', filename)
+            self.download_gh_file(dl_filename, api_url)
+
+    def update(self, tool):
+        mods = PipelineModules()
+        mods.get_modules_file_tree()
+
+    def remove(self, tool):
+        pass
+
+    def check_modules(self):
+        pass
+
+    def fix_modules(self):
+        pass
+
+
+    def get_modules_file_tree(self):
+        """
+        Fetch the file list from nf-core/modules, using the GitHub API
+
+        Sets self.modules_file_tree
+             self.modules_current_hash
+             self.modules_avail_tool_names
+        """
+        r = requests.get("https://api.github.com/repos/nf-core/modules/git/trees/master?recursive=1")
+        if r.status_code != 200:
+            raise SystemError("Could not fetch nf-core/modules tree: {}".format(r.status_code))
+
+        result = r.json()
+        assert result['truncated'] == False
+
+        self.modules_current_hash = result['sha']
+        self.modules_file_tree = result['tree']
+        for f in result['tree']:
+            if f['path'].startswith('tools/') and f['path'].count('/') == 1:
+                self.modules_avail_tool_names.append(f['path'].replace('tools/', ''))
+
+    def get_tool_file_urls(self, tool):
+        """Fetch list of URLs for a specific tool
+
+        Takes the name of a tool and iterates over the GitHub nf-core/modules file tree.
+        Loops over items that are prefixed with the path 'tools/<tool_name>' and ignores
+        anything that's not a blob.
+
+        Returns a dictionary with keys as filenames and values as GitHub API URIs.
+        These can be used to then download file contents.
+
+        Args:
+            tool (string): Name of tool for which to fetch a set of URLs
+
+        Returns:
+            dict: Set of files and associated URLs as follows:
+
+            {
+                'tools/fastqc/main.nf': 'https://api.github.com/repos/nf-core/modules/git/blobs/65ba598119206a2b851b86a9b5880b5476e263c3',
+                'tools/fastqc/meta.yml': 'https://api.github.com/repos/nf-core/modules/git/blobs/0d5afc23ba44d44a805c35902febc0a382b17651'
+            }
+        """
+        results = {}
+        for f in self.modules_file_tree:
+            if f['path'].startswith('tools/{}'.format(tool)) and f['type'] == 'blob':
+                results[f['path']] = f['url']
+        return results
+
+    def download_gh_file(self, dl_filename, api_url):
+        """Download a file from GitHub using the GitHub API
+
+        Args:
+            dl_filename (string): Path to save file to
+            api_url (string): GitHub API URL for file
+
+        Raises:
+            If a problem, raises an error
+        """
+
+        # Make target directory if it doesn't already exist
+        dl_directory = os.path.dirname(dl_filename)
+        if not os.path.exists(dl_directory):
+            os.makedirs(dl_directory)
+
+        # Call the GitHub API
+        r = requests.get(api_url)
+        if r.status_code != 200:
+            raise SystemError("Could not fetch nf-core/modules file: {}\n {}".format(r.status_code, api_url))
+        result = r.json()
+        file_contents = base64.b64decode(result['content'])
+
+        # Write the file contents
+        with open(dl_filename, 'wb') as fh:
+            fh.write(file_contents)
diff --git a/scripts/nf-core b/scripts/nf-core
index ee8017c1e5..bd43476c2f 100755
--- a/scripts/nf-core
+++ b/scripts/nf-core
@@ -346,29 +346,62 @@ def sync(pipeline_dir, make_template_branch, from_branch, pull_request, username
 
 ## nf-core module subcommands
 @nf_core_cli.group(cls=CustomHelpOrder)
-def module():
+def modules():
     """ Manage DSL 2 module imports """
     pass
 
-@module.command(help_priority=1)
-def install():
+@modules.command(help_priority=1)
+def list():
+    """ List available tools """
+    mods = nf_core.modules.PipelineModules()
+    mods.list_modules()
+
+@modules.command(help_priority=2)
+@click.argument(
+    'tool',
+    type = str,
+    required = True,
+    metavar = "<tool name>"
+)
+def install(tool):
     """ Install a DSL2 module """
-    pass
+    mods = nf_core.modules.PipelineModules()
+    mods.install(tool)
 
-@module.command(help_priority=2)
-def remove():
+@modules.command(help_priority=3)
+@click.argument(
+    'tool',
+    type = str,
+    metavar = "<tool name>"
+)
+def update(tool):
+    """ Update one or all DSL2 modules """
+    mods = nf_core.modules.PipelineModules()
+    mods.update(tool)
+
+@modules.command(help_priority=4)
+@click.argument(
+    'tool',
+    type = str,
+    required = True,
+    metavar = "<tool name>"
+)
+def remove(tool):
     """ Remove a DSL2 module """
-    pass
+    mods = nf_core.modules.PipelineModules()
+    mods.remove(tool)
 
-@module.command(help_priority=3)
+@modules.command(help_priority=5)
 def check():
     """ Check that imported module code has not been modified """
-    pass
+    mods = nf_core.modules.PipelineModules()
+    mods.check_modules()
 
-@module.command(help_priority=4)
+@modules.command(help_priority=6)
 def fix():
     """ Replace imported module code with a freshly downloaded copy """
-    pass
+    mods = nf_core.modules.PipelineModules()
+    mods.fix_modules()
 
 if __name__ == '__main__':
     click.echo(click.style("\n                                          ,--.", fg='green')+click.style("/",fg='black')+click.style(",-.", fg='green'), err=True)

From d1d7a9e8bf171ea0d1d6994f0abb841c475b3f30 Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Tue, 2 Jun 2020 18:18:16 +0200
Subject: [PATCH 03/14] nf-core modules - add option for all subcommands to
 work with any repo or branch

---
 nf_core/modules.py | 48 +++++++++++++++++++++++++-----------------
 scripts/nf-core    | 52 +++++++++++++++++++++++++++++++++-------------
 2 files changed, 67 insertions(+), 33 deletions(-)

diff --git a/nf_core/modules.py b/nf_core/modules.py
index 398066f904..859e40a45b 100644
--- a/nf_core/modules.py
+++ b/nf_core/modules.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 """
-Code to handle DSL2 module imports from nf-core/modules
+Code to handle DSL2 module imports from a GitHub repository
 """
 
 from __future__ import print_function
@@ -12,13 +12,25 @@
 import sys
 import tempfile
 
+class ModulesRepo(object):
+    """
+    An object to store details about the repository being used for modules.
+
+    Used by the `nf-core modules` top-level command with -r and -b flags,
+    so that this can be used in the same way by all sucommands.
+    """
+
+    def __init__(self, repo='nf-core/modules', branch='master'):
+        self.name = repo
+        self.branch = branch
 
 class PipelineModules(object):
 
-    def __init__(self):
+    def __init__(self, repo_obj):
         """
         Initialise the PipelineModules object
         """
+        self.repo = repo_obj
         self.pipeline_dir = os.getcwd()
         self.modules_file_tree = {}
         self.modules_current_hash = None
@@ -27,25 +39,23 @@ def __init__(self):
 
     def list_modules(self):
         """
-        Get available tool names from GitHub tree for nf-core/modules
+        Get available tool names from GitHub tree for repo
         and print as list to stdout
         """
-        mods = PipelineModules()
-        mods.get_modules_file_tree()
-        logging.info("Tools available from nf-core/modules:\n")
+        self.get_modules_file_tree()
+        logging.info("Tools available from {}:\n".format(self.repo.name))
         # Print results to stdout
-        print("\n".join(mods.modules_avail_tool_names))
+        print("\n".join(self.modules_avail_tool_names))
 
     def install(self, tool):
-        mods = PipelineModules()
-        mods.get_modules_file_tree()
+        self.get_modules_file_tree()
 
         # Check that the supplied name is an available tool
-        if tool not in mods.modules_avail_tool_names:
+        if tool not in self.modules_avail_tool_names:
             logging.error("Tool '{}' not found in list of available modules.".format(tool))
             logging.info("Use the command 'nf-core modules list' to view available tools")
             return
-        logging.debug("Installing tool '{}' at modules hash {}".format(tool, mods.modules_current_hash))
+        logging.debug("Installing tool '{}' at modules hash {}".format(tool, self.modules_current_hash))
 
         # Check that we don't already have a folder for this tool
         tool_dir = os.path.join(self.pipeline_dir, 'modules', 'tools', tool)
@@ -55,15 +65,14 @@ def install(self, tool):
             return
 
         # Download tool files
-        files = mods.get_tool_file_urls(tool)
+        files = self.get_tool_file_urls(tool)
         logging.debug("Fetching tool files:\n - {}".format("\n - ".join(files.keys())))
         for filename, api_url in files.items():
             dl_filename = os.path.join(self.pipeline_dir, 'modules', filename)
             self.download_gh_file(dl_filename, api_url)
 
     def update(self, tool):
-        mods = PipelineModules()
-        mods.get_modules_file_tree()
+        self.get_modules_file_tree()
 
     def remove(self, tool):
         pass
@@ -77,15 +86,16 @@ def fix_modules(self):
 
     def get_modules_file_tree(self):
         """
-        Fetch the file list from nf-core/modules, using the GitHub API
+        Fetch the file list from the repo, using the GitHub API
 
         Sets self.modules_file_tree
              self.modules_current_hash
              self.modules_avail_tool_names
         """
-        r = requests.get("https://api.github.com/repos/nf-core/modules/git/trees/master?recursive=1")
+        api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.repo.name, self.repo.branch)
+        r = requests.get(api_url)
         if r.status_code != 200:
-            raise SystemError("Could not fetch nf-core/modules tree: {}".format(r.status_code))
+            raise SystemError("Could not fetch {} tree: {}\n{}".format(self.repo.name, r.status_code, api_url))
 
         result = r.json()
         assert result['truncated'] == False
@@ -99,7 +109,7 @@ def get_modules_file_tree(self):
     def get_tool_file_urls(self, tool):
         """Fetch list of URLs for a specific tool
 
-        Takes the name of a tool and iterates over the GitHub nf-core/modules file tree.
+        Takes the name of a tool and iterates over the GitHub repo file tree.
         Loops over items that are prefixed with the path 'tools/<tool_name>' and ignores
         anything that's not a blob.
 
@@ -142,7 +152,7 @@ def download_gh_file(self, dl_filename, api_url):
         # Call the GitHub API
         r = requests.get(api_url)
         if r.status_code != 200:
-            raise SystemError("Could not fetch nf-core/modules file: {}\n {}".format(r.status_code, api_url))
+            raise SystemError("Could not fetch {} file: {}\n {}".format(self.repo.name, r.status_code, api_url))
         result = r.json()
         file_contents = base64.b64decode(result['content'])
 
diff --git a/scripts/nf-core b/scripts/nf-core
index 67854b0a79..6839b0d730 100755
--- a/scripts/nf-core
+++ b/scripts/nf-core
@@ -459,61 +459,85 @@ def sync(pipeline_dir, make_template_branch, from_branch, pull_request, username
 
 ## nf-core module subcommands
 @nf_core_cli.group(cls=CustomHelpOrder)
-def modules():
+@click.option(
+    '-r', '--repository',
+    type = str,
+    default = 'nf-core',
+    help = 'GitHub repository name.'
+)
+@click.option(
+    '-b', '--branch',
+    type = str,
+    default = 'master',
+    help = 'The git branch to use.'
+)
+@click.pass_context
+def modules(ctx, repository, branch):
     """ Manage DSL 2 module imports """
-    pass
+    # ensure that ctx.obj exists and is a dict (in case `cli()` is called
+    # by means other than the `if` block below)
+    ctx.ensure_object(dict)
+
+    # Make repository object to pass to subcommands
+    ctx.obj['repo_obj'] = nf_core.modules.ModulesRepo(repository, branch)
 
 @modules.command(help_priority=1)
-def list():
+@click.pass_context
+def list(ctx):
     """ List available tools """
-    mods = nf_core.modules.PipelineModules()
+    mods = nf_core.modules.PipelineModules(ctx.obj['repo_obj'])
     mods.list_modules()
 
 @modules.command(help_priority=2)
+@click.pass_context
 @click.argument(
     'tool',
     type = str,
     required = True,
     metavar = "<tool name>"
 )
-def install(tool):
+def install(ctx, tool):
     """ Install a DSL2 module """
-    mods = nf_core.modules.PipelineModules()
+    mods = nf_core.modules.PipelineModules(ctx.obj['repo_obj'])
     mods.install(tool)
 
 @modules.command(help_priority=3)
+@click.pass_context
 @click.argument(
     'tool',
     type = str,
     metavar = "<tool name>"
 )
-def update(tool):
+def update(ctx, tool):
     """ Update one or all DSL2 modules """
-    mods = nf_core.modules.PipelineModules()
+    mods = nf_core.modules.PipelineModules(ctx.obj['repo_obj'])
     mods.update(tool)
 
 @modules.command(help_priority=4)
+@click.pass_context
 @click.argument(
     'tool',
     type = str,
     required = True,
     metavar = "<tool name>"
 )
-def remove(tool):
+def remove(ctx, tool):
     """ Remove a DSL2 module """
-    mods = nf_core.modules.PipelineModules()
+    mods = nf_core.modules.PipelineModules(ctx.obj['repo_obj'])
     mods.remove(tool)
 
 @modules.command(help_priority=5)
-def check():
+@click.pass_context
+def check(ctx):
     """ Check that imported module code has not been modified """
-    mods = nf_core.modules.PipelineModules()
+    mods = nf_core.modules.PipelineModules(ctx.obj['repo_obj'])
     mods.check_modules()
 
 @modules.command(help_priority=6)
-def fix():
+@click.pass_context
+def fix(ctx):
     """ Replace imported module code with a freshly downloaded copy """
-    mods = nf_core.modules.PipelineModules()
+    mods = nf_core.modules.PipelineModules(ctx.obj['repo_obj'])
     mods.fix_modules()
 
 if __name__ == '__main__':

From f97d5dc367cbceda7723c05fab8d670fb4b8ce08 Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Tue, 2 Jun 2020 18:26:22 +0200
Subject: [PATCH 04/14] A little tidying

---
 nf_core/modules.py | 23 +++++++++++++++++------
 scripts/nf-core    |  2 +-
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/nf_core/modules.py b/nf_core/modules.py
index 859e40a45b..014e8127ee 100644
--- a/nf_core/modules.py
+++ b/nf_core/modules.py
@@ -43,9 +43,13 @@ def list_modules(self):
         and print as list to stdout
         """
         self.get_modules_file_tree()
-        logging.info("Tools available from {}:\n".format(self.repo.name))
-        # Print results to stdout
-        print("\n".join(self.modules_avail_tool_names))
+
+        if len(self.modules_avail_tool_names) > 0:
+            logging.info("Tools available from {} ({}):\n".format(self.repo.name, self.repo.branch))
+            # Print results to stdout
+            print("\n".join(self.modules_avail_tool_names))
+        else:
+            logging.info("No available tools found in {} ({}):\n".format(self.repo.name, self.repo.branch))
 
     def install(self, tool):
         self.get_modules_file_tree()
@@ -72,15 +76,19 @@ def install(self, tool):
             self.download_gh_file(dl_filename, api_url)
 
     def update(self, tool):
-        self.get_modules_file_tree()
+        logging.error("This command is not yet implemented")
+        pass
 
     def remove(self, tool):
+        logging.error("This command is not yet implemented")
         pass
 
     def check_modules(self):
+        logging.error("This command is not yet implemented")
         pass
 
     def fix_modules(self):
+        logging.error("This command is not yet implemented")
         pass
 
 
@@ -94,8 +102,11 @@ def get_modules_file_tree(self):
         """
         api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.repo.name, self.repo.branch)
         r = requests.get(api_url)
-        if r.status_code != 200:
-            raise SystemError("Could not fetch {} tree: {}\n{}".format(self.repo.name, r.status_code, api_url))
+        if r.status_code == 404:
+            logging.error("Repository / branch not found: {} ({})\n{}".format(self.repo.name, self.repo.branch, api_url))
+            sys.exit(1)
+        elif r.status_code != 200:
+            raise SystemError("Could not fetch {} ({}) tree: {}\n{}".format(self.repo.name, self.repo.branch, r.status_code, api_url))
 
         result = r.json()
         assert result['truncated'] == False
diff --git a/scripts/nf-core b/scripts/nf-core
index 6839b0d730..4482dd37ed 100755
--- a/scripts/nf-core
+++ b/scripts/nf-core
@@ -462,7 +462,7 @@ def sync(pipeline_dir, make_template_branch, from_branch, pull_request, username
 @click.option(
     '-r', '--repository',
     type = str,
-    default = 'nf-core',
+    default = 'nf-core/modules',
     help = 'GitHub repository name.'
 )
 @click.option(

From 8f1dcbea0269431e03ca2296f8d7e7f75fd58ca2 Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Sat, 11 Jul 2020 15:04:33 +0200
Subject: [PATCH 05/14] Black: modules.py

---
 nf_core/modules.py | 41 ++++++++++++++++++++++-------------------
 1 file changed, 22 insertions(+), 19 deletions(-)

diff --git a/nf_core/modules.py b/nf_core/modules.py
index 014e8127ee..90ff432962 100644
--- a/nf_core/modules.py
+++ b/nf_core/modules.py
@@ -12,6 +12,7 @@
 import sys
 import tempfile
 
+
 class ModulesRepo(object):
     """
     An object to store details about the repository being used for modules.
@@ -20,12 +21,12 @@ class ModulesRepo(object):
     so that this can be used in the same way by all sucommands.
     """
 
-    def __init__(self, repo='nf-core/modules', branch='master'):
+    def __init__(self, repo="nf-core/modules", branch="master"):
         self.name = repo
         self.branch = branch
 
-class PipelineModules(object):
 
+class PipelineModules(object):
     def __init__(self, repo_obj):
         """
         Initialise the PipelineModules object
@@ -36,7 +37,6 @@ def __init__(self, repo_obj):
         self.modules_current_hash = None
         self.modules_avail_tool_names = []
 
-
     def list_modules(self):
         """
         Get available tool names from GitHub tree for repo
@@ -62,8 +62,8 @@ def install(self, tool):
         logging.debug("Installing tool '{}' at modules hash {}".format(tool, self.modules_current_hash))
 
         # Check that we don't already have a folder for this tool
-        tool_dir = os.path.join(self.pipeline_dir, 'modules', 'tools', tool)
-        if(os.path.exists(tool_dir)):
+        tool_dir = os.path.join(self.pipeline_dir, "modules", "tools", tool)
+        if os.path.exists(tool_dir):
             logging.error("Tool directory already exists: {}".format(tool_dir))
             logging.info("To update an existing tool, use the commands 'nf-core update' or 'nf-core fix'")
             return
@@ -72,7 +72,7 @@ def install(self, tool):
         files = self.get_tool_file_urls(tool)
         logging.debug("Fetching tool files:\n - {}".format("\n - ".join(files.keys())))
         for filename, api_url in files.items():
-            dl_filename = os.path.join(self.pipeline_dir, 'modules', filename)
+            dl_filename = os.path.join(self.pipeline_dir, "modules", filename)
             self.download_gh_file(dl_filename, api_url)
 
     def update(self, tool):
@@ -91,7 +91,6 @@ def fix_modules(self):
         logging.error("This command is not yet implemented")
         pass
 
-
     def get_modules_file_tree(self):
         """
         Fetch the file list from the repo, using the GitHub API
@@ -103,19 +102,23 @@ def get_modules_file_tree(self):
         api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.repo.name, self.repo.branch)
         r = requests.get(api_url)
         if r.status_code == 404:
-            logging.error("Repository / branch not found: {} ({})\n{}".format(self.repo.name, self.repo.branch, api_url))
+            logging.error(
+                "Repository / branch not found: {} ({})\n{}".format(self.repo.name, self.repo.branch, api_url)
+            )
             sys.exit(1)
         elif r.status_code != 200:
-            raise SystemError("Could not fetch {} ({}) tree: {}\n{}".format(self.repo.name, self.repo.branch, r.status_code, api_url))
+            raise SystemError(
+                "Could not fetch {} ({}) tree: {}\n{}".format(self.repo.name, self.repo.branch, r.status_code, api_url)
+            )
 
         result = r.json()
-        assert result['truncated'] == False
+        assert result["truncated"] == False
 
-        self.modules_current_hash = result['sha']
-        self.modules_file_tree = result['tree']
-        for f in result['tree']:
-            if f['path'].startswith('tools/') and f['path'].count('/') == 1:
-                self.modules_avail_tool_names.append(f['path'].replace('tools/', ''))
+        self.modules_current_hash = result["sha"]
+        self.modules_file_tree = result["tree"]
+        for f in result["tree"]:
+            if f["path"].startswith("tools/") and f["path"].count("/") == 1:
+                self.modules_avail_tool_names.append(f["path"].replace("tools/", ""))
 
     def get_tool_file_urls(self, tool):
         """Fetch list of URLs for a specific tool
@@ -140,8 +143,8 @@ def get_tool_file_urls(self, tool):
         """
         results = {}
         for f in self.modules_file_tree:
-            if f['path'].startswith('tools/{}'.format(tool)) and f['type'] == 'blob':
-                results[f['path']] = f['url']
+            if f["path"].startswith("tools/{}".format(tool)) and f["type"] == "blob":
+                results[f["path"]] = f["url"]
         return results
 
     def download_gh_file(self, dl_filename, api_url):
@@ -165,8 +168,8 @@ def download_gh_file(self, dl_filename, api_url):
         if r.status_code != 200:
             raise SystemError("Could not fetch {} file: {}\n {}".format(self.repo.name, r.status_code, api_url))
         result = r.json()
-        file_contents = base64.b64decode(result['content'])
+        file_contents = base64.b64decode(result["content"])
 
         # Write the file contents
-        with open(dl_filename, 'wb') as fh:
+        with open(dl_filename, "wb") as fh:
             fh.write(file_contents)

From e91d2e0a9338b60e98971e3718844d9deeb17fcd Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Sat, 11 Jul 2020 15:05:37 +0200
Subject: [PATCH 06/14] Module import: rename tools to software

See nf-core/modules#7
---
 nf_core/modules.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/nf_core/modules.py b/nf_core/modules.py
index 90ff432962..a089c2402d 100644
--- a/nf_core/modules.py
+++ b/nf_core/modules.py
@@ -45,11 +45,11 @@ def list_modules(self):
         self.get_modules_file_tree()
 
         if len(self.modules_avail_tool_names) > 0:
-            logging.info("Tools available from {} ({}):\n".format(self.repo.name, self.repo.branch))
+            logging.info("Software available from {} ({}):\n".format(self.repo.name, self.repo.branch))
             # Print results to stdout
             print("\n".join(self.modules_avail_tool_names))
         else:
-            logging.info("No available tools found in {} ({}):\n".format(self.repo.name, self.repo.branch))
+            logging.info("No available software found in {} ({}):\n".format(self.repo.name, self.repo.branch))
 
     def install(self, tool):
         self.get_modules_file_tree()
@@ -57,12 +57,12 @@ def install(self, tool):
         # Check that the supplied name is an available tool
         if tool not in self.modules_avail_tool_names:
             logging.error("Tool '{}' not found in list of available modules.".format(tool))
-            logging.info("Use the command 'nf-core modules list' to view available tools")
+            logging.info("Use the command 'nf-core modules list' to view available software")
             return
         logging.debug("Installing tool '{}' at modules hash {}".format(tool, self.modules_current_hash))
 
         # Check that we don't already have a folder for this tool
-        tool_dir = os.path.join(self.pipeline_dir, "modules", "tools", tool)
+        tool_dir = os.path.join(self.pipeline_dir, "modules", "software", tool)
         if os.path.exists(tool_dir):
             logging.error("Tool directory already exists: {}".format(tool_dir))
             logging.info("To update an existing tool, use the commands 'nf-core update' or 'nf-core fix'")
@@ -117,14 +117,14 @@ def get_modules_file_tree(self):
         self.modules_current_hash = result["sha"]
         self.modules_file_tree = result["tree"]
         for f in result["tree"]:
-            if f["path"].startswith("tools/") and f["path"].count("/") == 1:
-                self.modules_avail_tool_names.append(f["path"].replace("tools/", ""))
+            if f["path"].startswith("software/") and f["path"].count("/") == 1:
+                self.modules_avail_tool_names.append(f["path"].replace("software/", ""))
 
     def get_tool_file_urls(self, tool):
         """Fetch list of URLs for a specific tool
 
         Takes the name of a tool and iterates over the GitHub repo file tree.
-        Loops over items that are prefixed with the path 'tools/<tool_name>' and ignores
+        Loops over items that are prefixed with the path 'software/<tool_name>' and ignores
         anything that's not a blob.
 
         Returns a dictionary with keys as filenames and values as GitHub API URIs.
@@ -137,13 +137,13 @@ def get_tool_file_urls(self, tool):
             dict: Set of files and associated URLs as follows:
 
             {
-                'tools/fastqc/main.nf': 'https://api.github.com/repos/nf-core/modules/git/blobs/65ba598119206a2b851b86a9b5880b5476e263c3',
-                'tools/fastqc/meta.yml': 'https://api.github.com/repos/nf-core/modules/git/blobs/0d5afc23ba44d44a805c35902febc0a382b17651'
+                'software/fastqc/main.nf': 'https://api.github.com/repos/nf-core/modules/git/blobs/65ba598119206a2b851b86a9b5880b5476e263c3',
+                'software/fastqc/meta.yml': 'https://api.github.com/repos/nf-core/modules/git/blobs/0d5afc23ba44d44a805c35902febc0a382b17651'
             }
         """
         results = {}
         for f in self.modules_file_tree:
-            if f["path"].startswith("tools/{}".format(tool)) and f["type"] == "blob":
+            if f["path"].startswith("software/{}".format(tool)) and f["type"] == "blob":
                 results[f["path"]] = f["url"]
         return results
 

From 9345b9bed173bd0a8101ba9c5b6079a2de76df27 Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Sat, 11 Jul 2020 16:48:31 +0200
Subject: [PATCH 07/14] Re-order nf-core help subcommands

---
 nf_core/__main__.py | 152 ++++++++++++++++++++++++--------------------
 1 file changed, 83 insertions(+), 69 deletions(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index 7c390623b5..52c16d7510 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -68,6 +68,20 @@ def decorator(f):
 
         return decorator
 
+    def group(self, *args, **kwargs):
+        """Behaves the same as `click.Group.group()` except capture
+        a priority for listing command names in help.
+        """
+        help_priority = kwargs.pop("help_priority", 1000)
+        help_priorities = self.help_priorities
+
+        def decorator(f):
+            cmd = super(CustomHelpOrder, self).command(*args, **kwargs)(f)
+            help_priorities[cmd.name] = help_priority
+            return cmd
+
+        return decorator
+
 
 @click.group(cls=CustomHelpOrder)
 @click.version_option(nf_core.__version__)
@@ -253,8 +267,74 @@ def lint(pipeline_dir, release, markdown, json):
         sys.exit(1)
 
 
+## nf-core module subcommands
+@nf_core_cli.group(cls=CustomHelpOrder, help_priority=7)
+@click.option("-r", "--repository", type=str, default="nf-core/modules", help="GitHub repository name.")
+@click.option("-b", "--branch", type=str, default="master", help="The git branch to use.")
+@click.pass_context
+def modules(ctx, repository, branch):
+    """ Manage DSL 2 module imports """
+    # ensure that ctx.obj exists and is a dict (in case `cli()` is called
+    # by means other than the `if` block below)
+    ctx.ensure_object(dict)
+
+    # Make repository object to pass to subcommands
+    ctx.obj["repo_obj"] = nf_core.modules.ModulesRepo(repository, branch)
+
+
+@modules.command(help_priority=1)
+@click.pass_context
+def list(ctx):
+    """ List available tools """
+    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
+    mods.list_modules()
+
+
+@modules.command(help_priority=2)
+@click.pass_context
+@click.argument("tool", type=str, required=True, metavar="<tool name>")
+def install(ctx, tool):
+    """ Install a DSL2 module """
+    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
+    mods.install(tool)
+
+
+@modules.command(help_priority=3)
+@click.pass_context
+@click.argument("tool", type=str, metavar="<tool name>")
+def update(ctx, tool):
+    """ Update one or all DSL2 modules """
+    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
+    mods.update(tool)
+
+
+@modules.command(help_priority=4)
+@click.pass_context
+@click.argument("tool", type=str, required=True, metavar="<tool name>")
+def remove(ctx, tool):
+    """ Remove a DSL2 module """
+    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
+    mods.remove(tool)
+
+
+@modules.command(help_priority=5)
+@click.pass_context
+def check(ctx):
+    """ Check that imported module code has not been modified """
+    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
+    mods.check_modules()
+
+
+@modules.command(help_priority=6)
+@click.pass_context
+def fix(ctx):
+    """ Replace imported module code with a freshly downloaded copy """
+    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
+    mods.fix_modules()
+
+
 ## nf-core schema subcommands
-@nf_core_cli.group(cls=CustomHelpOrder)
+@nf_core_cli.group(cls=CustomHelpOrder, help_priority=8)
 def schema():
     """
     Suite of tools for developers to manage pipeline schema.
@@ -341,7 +421,7 @@ def lint(schema_path):
         sys.exit(1)
 
 
-@nf_core_cli.command("bump-version", help_priority=7)
+@nf_core_cli.command("bump-version", help_priority=9)
 @click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="<pipeline directory>")
 @click.argument("new_version", required=True, metavar="<new version>")
 @click.option(
@@ -375,7 +455,7 @@ def bump_version(pipeline_dir, new_version, nextflow):
         nf_core.bump_version.bump_nextflow_version(lint_obj, new_version)
 
 
-@nf_core_cli.command("sync", help_priority=8)
+@nf_core_cli.command("sync", help_priority=10)
 @click.argument("pipeline_dir", type=click.Path(exists=True), nargs=-1, metavar="<pipeline directory>")
 @click.option(
     "-t", "--make-template-branch", is_flag=True, default=False, help="Create a TEMPLATE branch if none is found."
@@ -420,71 +500,5 @@ def sync(pipeline_dir, make_template_branch, from_branch, pull_request, username
             sys.exit(1)
 
 
-## nf-core module subcommands
-@nf_core_cli.group(cls=CustomHelpOrder)
-@click.option("-r", "--repository", type=str, default="nf-core/modules", help="GitHub repository name.")
-@click.option("-b", "--branch", type=str, default="master", help="The git branch to use.")
-@click.pass_context
-def modules(ctx, repository, branch):
-    """ Manage DSL 2 module imports """
-    # ensure that ctx.obj exists and is a dict (in case `cli()` is called
-    # by means other than the `if` block below)
-    ctx.ensure_object(dict)
-
-    # Make repository object to pass to subcommands
-    ctx.obj["repo_obj"] = nf_core.modules.ModulesRepo(repository, branch)
-
-
-@modules.command(help_priority=1)
-@click.pass_context
-def list(ctx):
-    """ List available tools """
-    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
-    mods.list_modules()
-
-
-@modules.command(help_priority=2)
-@click.pass_context
-@click.argument("tool", type=str, required=True, metavar="<tool name>")
-def install(ctx, tool):
-    """ Install a DSL2 module """
-    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
-    mods.install(tool)
-
-
-@modules.command(help_priority=3)
-@click.pass_context
-@click.argument("tool", type=str, metavar="<tool name>")
-def update(ctx, tool):
-    """ Update one or all DSL2 modules """
-    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
-    mods.update(tool)
-
-
-@modules.command(help_priority=4)
-@click.pass_context
-@click.argument("tool", type=str, required=True, metavar="<tool name>")
-def remove(ctx, tool):
-    """ Remove a DSL2 module """
-    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
-    mods.remove(tool)
-
-
-@modules.command(help_priority=5)
-@click.pass_context
-def check(ctx):
-    """ Check that imported module code has not been modified """
-    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
-    mods.check_modules()
-
-
-@modules.command(help_priority=6)
-@click.pass_context
-def fix(ctx):
-    """ Replace imported module code with a freshly downloaded copy """
-    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
-    mods.fix_modules()
-
-
 if __name__ == "__main__":
     run_nf_core()

From 51aaa502fa3dc429127bc866b25eae8e03887717 Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Sat, 11 Jul 2020 16:56:54 +0200
Subject: [PATCH 08/14] Modules: Improve help text.

Also remove fix subcommand, better done with update --force
---
 nf_core/__main__.py | 51 +++++++++++++++++++++++++++++++++------------
 nf_core/modules.py  |  4 ----
 2 files changed, 38 insertions(+), 17 deletions(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index 52c16d7510..e0a34c92da 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -273,7 +273,11 @@ def lint(pipeline_dir, release, markdown, json):
 @click.option("-b", "--branch", type=str, default="master", help="The git branch to use.")
 @click.pass_context
 def modules(ctx, repository, branch):
-    """ Manage DSL 2 module imports """
+    """
+    Work with the nf-core/modules software wrappers.
+
+    Tools to manage DSL 2 nf-core/modules software wrapper imports.
+    """
     # ensure that ctx.obj exists and is a dict (in case `cli()` is called
     # by means other than the `if` block below)
     ctx.ensure_object(dict)
@@ -285,7 +289,11 @@ def modules(ctx, repository, branch):
 @modules.command(help_priority=1)
 @click.pass_context
 def list(ctx):
-    """ List available tools """
+    """
+    List available software modules.
+
+    Lists all currently available software wrappers in the nf-core/modules repository.
+    """
     mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
     mods.list_modules()
 
@@ -294,7 +302,12 @@ def list(ctx):
 @click.pass_context
 @click.argument("tool", type=str, required=True, metavar="<tool name>")
 def install(ctx, tool):
-    """ Install a DSL2 module """
+    """
+    Add a DSL2 software wrapper module to a pipeline.
+
+    Given a software name, finds the relevant files in nf-core/modules
+    and copies to the pipeline along with associated metadata.
+    """
     mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
     mods.install(tool)
 
@@ -302,8 +315,17 @@ def install(ctx, tool):
 @modules.command(help_priority=3)
 @click.pass_context
 @click.argument("tool", type=str, metavar="<tool name>")
+# --force - overwrite files even if no update found
 def update(ctx, tool):
-    """ Update one or all DSL2 modules """
+    """
+    Update one or all software wrapper modules.
+
+    Compares a currently installed module against what is available in nf-core/modules.
+    Fetchs files and updates all relevant files for that software wrapper.
+
+    If no module name is specified, loops through all currently installed modules.
+    If no version is specified, looks for the latest available version on nf-core/modules.
+    """
     mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
     mods.update(tool)
 
@@ -312,7 +334,9 @@ def update(ctx, tool):
 @click.pass_context
 @click.argument("tool", type=str, required=True, metavar="<tool name>")
 def remove(ctx, tool):
-    """ Remove a DSL2 module """
+    """
+    Remove a software wrapper from a pipeline.
+    """
     mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
     mods.remove(tool)
 
@@ -320,17 +344,18 @@ def remove(ctx, tool):
 @modules.command(help_priority=5)
 @click.pass_context
 def check(ctx):
-    """ Check that imported module code has not been modified """
-    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
-    mods.check_modules()
+    """
+    Check that imported module code has not been modified.
 
+    Compares a software module against the copy on nf-core/modules.
+    If any local modifications are found, the command logs an error
+    and exits with a non-zero exit code.
 
-@modules.command(help_priority=6)
-@click.pass_context
-def fix(ctx):
-    """ Replace imported module code with a freshly downloaded copy """
+    Use by the lint tests and automated CI to check that centralised
+    software wrapper code is only modified in the central repository.
+    """
     mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
-    mods.fix_modules()
+    mods.check_modules()
 
 
 ## nf-core schema subcommands
diff --git a/nf_core/modules.py b/nf_core/modules.py
index a089c2402d..698a7f80e1 100644
--- a/nf_core/modules.py
+++ b/nf_core/modules.py
@@ -87,10 +87,6 @@ def check_modules(self):
         logging.error("This command is not yet implemented")
         pass
 
-    def fix_modules(self):
-        logging.error("This command is not yet implemented")
-        pass
-
     def get_modules_file_tree(self):
         """
         Fetch the file list from the repo, using the GitHub API

From d7ddaafa7482fc0c7ccf7955f61a2b1952835358 Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Sat, 11 Jul 2020 16:59:38 +0200
Subject: [PATCH 09/14] Modules: continue removing use of 'tool' in code

---
 nf_core/modules.py | 54 +++++++++++++++++++++++-----------------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/nf_core/modules.py b/nf_core/modules.py
index 698a7f80e1..c74ba53aad 100644
--- a/nf_core/modules.py
+++ b/nf_core/modules.py
@@ -35,51 +35,51 @@ def __init__(self, repo_obj):
         self.pipeline_dir = os.getcwd()
         self.modules_file_tree = {}
         self.modules_current_hash = None
-        self.modules_avail_tool_names = []
+        self.modules_avail_module_names = []
 
     def list_modules(self):
         """
-        Get available tool names from GitHub tree for repo
+        Get available module names from GitHub tree for repo
         and print as list to stdout
         """
         self.get_modules_file_tree()
 
-        if len(self.modules_avail_tool_names) > 0:
+        if len(self.modules_avail_module_names) > 0:
             logging.info("Software available from {} ({}):\n".format(self.repo.name, self.repo.branch))
             # Print results to stdout
-            print("\n".join(self.modules_avail_tool_names))
+            print("\n".join(self.modules_avail_module_names))
         else:
             logging.info("No available software found in {} ({}):\n".format(self.repo.name, self.repo.branch))
 
-    def install(self, tool):
+    def install(self, module):
         self.get_modules_file_tree()
 
-        # Check that the supplied name is an available tool
-        if tool not in self.modules_avail_tool_names:
-            logging.error("Tool '{}' not found in list of available modules.".format(tool))
+        # Check that the supplied name is an available module
+        if module not in self.modules_avail_module_names:
+            logging.error("Module '{}' not found in list of available modules.".format(module))
             logging.info("Use the command 'nf-core modules list' to view available software")
             return
-        logging.debug("Installing tool '{}' at modules hash {}".format(tool, self.modules_current_hash))
+        logging.debug("Installing module '{}' at modules hash {}".format(module, self.modules_current_hash))
 
-        # Check that we don't already have a folder for this tool
-        tool_dir = os.path.join(self.pipeline_dir, "modules", "software", tool)
-        if os.path.exists(tool_dir):
-            logging.error("Tool directory already exists: {}".format(tool_dir))
-            logging.info("To update an existing tool, use the commands 'nf-core update' or 'nf-core fix'")
+        # Check that we don't already have a folder for this module
+        module_dir = os.path.join(self.pipeline_dir, "modules", "software", module)
+        if os.path.exists(module_dir):
+            logging.error("Module directory already exists: {}".format(module_dir))
+            logging.info("To update an existing module, use the commands 'nf-core update' or 'nf-core fix'")
             return
 
-        # Download tool files
-        files = self.get_tool_file_urls(tool)
-        logging.debug("Fetching tool files:\n - {}".format("\n - ".join(files.keys())))
+        # Download module files
+        files = self.get_module_file_urls(module)
+        logging.debug("Fetching module files:\n - {}".format("\n - ".join(files.keys())))
         for filename, api_url in files.items():
             dl_filename = os.path.join(self.pipeline_dir, "modules", filename)
             self.download_gh_file(dl_filename, api_url)
 
-    def update(self, tool):
+    def update(self, module):
         logging.error("This command is not yet implemented")
         pass
 
-    def remove(self, tool):
+    def remove(self, module):
         logging.error("This command is not yet implemented")
         pass
 
@@ -93,7 +93,7 @@ def get_modules_file_tree(self):
 
         Sets self.modules_file_tree
              self.modules_current_hash
-             self.modules_avail_tool_names
+             self.modules_avail_module_names
         """
         api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.repo.name, self.repo.branch)
         r = requests.get(api_url)
@@ -114,20 +114,20 @@ def get_modules_file_tree(self):
         self.modules_file_tree = result["tree"]
         for f in result["tree"]:
             if f["path"].startswith("software/") and f["path"].count("/") == 1:
-                self.modules_avail_tool_names.append(f["path"].replace("software/", ""))
+                self.modules_avail_module_names.append(f["path"].replace("software/", ""))
 
-    def get_tool_file_urls(self, tool):
-        """Fetch list of URLs for a specific tool
+    def get_module_file_urls(self, module):
+        """Fetch list of URLs for a specific module
 
-        Takes the name of a tool and iterates over the GitHub repo file tree.
-        Loops over items that are prefixed with the path 'software/<tool_name>' and ignores
+        Takes the name of a module and iterates over the GitHub repo file tree.
+        Loops over items that are prefixed with the path 'software/<module_name>' and ignores
         anything that's not a blob.
 
         Returns a dictionary with keys as filenames and values as GitHub API URIs.
         These can be used to then download file contents.
 
         Args:
-            tool (string): Name of tool for which to fetch a set of URLs
+            module (string): Name of module for which to fetch a set of URLs
 
         Returns:
             dict: Set of files and associated URLs as follows:
@@ -139,7 +139,7 @@ def get_tool_file_urls(self, tool):
         """
         results = {}
         for f in self.modules_file_tree:
-            if f["path"].startswith("software/{}".format(tool)) and f["type"] == "blob":
+            if f["path"].startswith("software/{}".format(module)) and f["type"] == "blob":
                 results[f["path"]] = f["url"]
         return results
 

From 813d0a6cf65e4c55d5e0aa9dfa3a53f093a6ab28 Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Sat, 11 Jul 2020 17:14:51 +0200
Subject: [PATCH 10/14] Get modules - allow nested module directory structure

---
 nf_core/modules.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/nf_core/modules.py b/nf_core/modules.py
index c74ba53aad..65cb349e6f 100644
--- a/nf_core/modules.py
+++ b/nf_core/modules.py
@@ -113,8 +113,9 @@ def get_modules_file_tree(self):
         self.modules_current_hash = result["sha"]
         self.modules_file_tree = result["tree"]
         for f in result["tree"]:
-            if f["path"].startswith("software/") and f["path"].count("/") == 1:
-                self.modules_avail_module_names.append(f["path"].replace("software/", ""))
+            if f["path"].startswith("software/") and f["path"].endswith("/main.nf") and "/test/" not in f["path"]:
+                # remove software/ and /main.nf
+                self.modules_avail_module_names.append(f["path"][9:-8])
 
     def get_module_file_urls(self, module):
         """Fetch list of URLs for a specific module

From 54f61649d0e2eee6bb3890d950602ae37dbe258d Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Mon, 13 Jul 2020 10:08:28 +0200
Subject: [PATCH 11/14] Clarify modules repo variable names.

---
 nf_core/__main__.py | 22 ++++++++++++++--------
 nf_core/modules.py  | 30 ++++++++++++++++++++----------
 2 files changed, 34 insertions(+), 18 deletions(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index e0a34c92da..dccea3753a 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -269,8 +269,14 @@ def lint(pipeline_dir, release, markdown, json):
 
 ## nf-core module subcommands
 @nf_core_cli.group(cls=CustomHelpOrder, help_priority=7)
-@click.option("-r", "--repository", type=str, default="nf-core/modules", help="GitHub repository name.")
-@click.option("-b", "--branch", type=str, default="master", help="The git branch to use.")
+@click.option(
+    "-r",
+    "--repository",
+    type=str,
+    default="nf-core/modules",
+    help="GitHub repository hosting software wrapper modules.",
+)
+@click.option("-b", "--branch", type=str, default="master", help="Modules GitHub repo git branch to use.")
 @click.pass_context
 def modules(ctx, repository, branch):
     """
@@ -283,7 +289,7 @@ def modules(ctx, repository, branch):
     ctx.ensure_object(dict)
 
     # Make repository object to pass to subcommands
-    ctx.obj["repo_obj"] = nf_core.modules.ModulesRepo(repository, branch)
+    ctx.obj["modules_repo_obj"] = nf_core.modules.ModulesRepo(repository, branch)
 
 
 @modules.command(help_priority=1)
@@ -294,7 +300,7 @@ def list(ctx):
 
     Lists all currently available software wrappers in the nf-core/modules repository.
     """
-    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
+    mods = nf_core.modules.PipelineModules(ctx.obj["modules_repo_obj"])
     mods.list_modules()
 
 
@@ -308,7 +314,7 @@ def install(ctx, tool):
     Given a software name, finds the relevant files in nf-core/modules
     and copies to the pipeline along with associated metadata.
     """
-    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
+    mods = nf_core.modules.PipelineModules(ctx.obj["modules_repo_obj"])
     mods.install(tool)
 
 
@@ -326,7 +332,7 @@ def update(ctx, tool):
     If no module name is specified, loops through all currently installed modules.
     If no version is specified, looks for the latest available version on nf-core/modules.
     """
-    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
+    mods = nf_core.modules.PipelineModules(ctx.obj["modules_repo_obj"])
     mods.update(tool)
 
 
@@ -337,7 +343,7 @@ def remove(ctx, tool):
     """
     Remove a software wrapper from a pipeline.
     """
-    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
+    mods = nf_core.modules.PipelineModules(ctx.obj["modules_repo_obj"])
     mods.remove(tool)
 
 
@@ -354,7 +360,7 @@ def check(ctx):
     Use by the lint tests and automated CI to check that centralised
     software wrapper code is only modified in the central repository.
     """
-    mods = nf_core.modules.PipelineModules(ctx.obj["repo_obj"])
+    mods = nf_core.modules.PipelineModules(ctx.obj["modules_repo_obj"])
     mods.check_modules()
 
 
diff --git a/nf_core/modules.py b/nf_core/modules.py
index 65cb349e6f..d8d83b6d31 100644
--- a/nf_core/modules.py
+++ b/nf_core/modules.py
@@ -27,11 +27,11 @@ def __init__(self, repo="nf-core/modules", branch="master"):
 
 
 class PipelineModules(object):
-    def __init__(self, repo_obj):
+    def __init__(self, modules_repo_obj):
         """
         Initialise the PipelineModules object
         """
-        self.repo = repo_obj
+        self.modules_repo = modules_repo_obj
         self.pipeline_dir = os.getcwd()
         self.modules_file_tree = {}
         self.modules_current_hash = None
@@ -45,20 +45,24 @@ def list_modules(self):
         self.get_modules_file_tree()
 
         if len(self.modules_avail_module_names) > 0:
-            logging.info("Software available from {} ({}):\n".format(self.repo.name, self.repo.branch))
+            logging.info("Software available from {} ({}):\n".format(self.modules_repo.name, self.modules_repo.branch))
             # Print results to stdout
             print("\n".join(self.modules_avail_module_names))
         else:
-            logging.info("No available software found in {} ({}):\n".format(self.repo.name, self.repo.branch))
+            logging.info(
+                "No available software found in {} ({}):\n".format(self.modules_repo.name, self.modules_repo.branch)
+            )
 
     def install(self, module):
+
+        # Get the available modules
         self.get_modules_file_tree()
 
         # Check that the supplied name is an available module
         if module not in self.modules_avail_module_names:
             logging.error("Module '{}' not found in list of available modules.".format(module))
             logging.info("Use the command 'nf-core modules list' to view available software")
-            return
+            return False
         logging.debug("Installing module '{}' at modules hash {}".format(module, self.modules_current_hash))
 
         # Check that we don't already have a folder for this module
@@ -66,7 +70,7 @@ def install(self, module):
         if os.path.exists(module_dir):
             logging.error("Module directory already exists: {}".format(module_dir))
             logging.info("To update an existing module, use the commands 'nf-core update' or 'nf-core fix'")
-            return
+            return False
 
         # Download module files
         files = self.get_module_file_urls(module)
@@ -95,16 +99,22 @@ def get_modules_file_tree(self):
              self.modules_current_hash
              self.modules_avail_module_names
         """
-        api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.repo.name, self.repo.branch)
+        api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(
+            self.modules_repo.name, self.modules_repo.branch
+        )
         r = requests.get(api_url)
         if r.status_code == 404:
             logging.error(
-                "Repository / branch not found: {} ({})\n{}".format(self.repo.name, self.repo.branch, api_url)
+                "Repository / branch not found: {} ({})\n{}".format(
+                    self.modules_repo.name, self.modules_repo.branch, api_url
+                )
             )
             sys.exit(1)
         elif r.status_code != 200:
             raise SystemError(
-                "Could not fetch {} ({}) tree: {}\n{}".format(self.repo.name, self.repo.branch, r.status_code, api_url)
+                "Could not fetch {} ({}) tree: {}\n{}".format(
+                    self.modules_repo.name, self.modules_repo.branch, r.status_code, api_url
+                )
             )
 
         result = r.json()
@@ -163,7 +173,7 @@ def download_gh_file(self, dl_filename, api_url):
         # Call the GitHub API
         r = requests.get(api_url)
         if r.status_code != 200:
-            raise SystemError("Could not fetch {} file: {}\n {}".format(self.repo.name, r.status_code, api_url))
+            raise SystemError("Could not fetch {} file: {}\n {}".format(self.modules_repo.name, r.status_code, api_url))
         result = r.json()
         file_contents = base64.b64decode(result["content"])
 

From 47393ca9e2df983f7d47a5703cb816a274333356 Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Mon, 13 Jul 2020 10:42:02 +0200
Subject: [PATCH 12/14] Modules install improvements

* Pass pipeline directory as an argument
* Don't copy test directory to  pipeline
---
 nf_core/__main__.py | 19 +++++++++++++------
 nf_core/modules.py  | 27 +++++++++++++++++++++------
 2 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index dccea3753a..7275fd6e5a 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -300,21 +300,25 @@ def list(ctx):
 
     Lists all currently available software wrappers in the nf-core/modules repository.
     """
-    mods = nf_core.modules.PipelineModules(ctx.obj["modules_repo_obj"])
+    mods = nf_core.modules.PipelineModules()
+    mods.modules_repo = ctx.obj["modules_repo_obj"]
     mods.list_modules()
 
 
 @modules.command(help_priority=2)
 @click.pass_context
+@click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="<pipeline directory>")
 @click.argument("tool", type=str, required=True, metavar="<tool name>")
-def install(ctx, tool):
+def install(ctx, pipeline_dir, tool):
     """
     Add a DSL2 software wrapper module to a pipeline.
 
     Given a software name, finds the relevant files in nf-core/modules
     and copies to the pipeline along with associated metadata.
     """
-    mods = nf_core.modules.PipelineModules(ctx.obj["modules_repo_obj"])
+    mods = nf_core.modules.PipelineModules()
+    mods.modules_repo = ctx.obj["modules_repo_obj"]
+    mods.pipeline_dir = pipeline_dir
     mods.install(tool)
 
 
@@ -332,7 +336,8 @@ def update(ctx, tool):
     If no module name is specified, loops through all currently installed modules.
     If no version is specified, looks for the latest available version on nf-core/modules.
     """
-    mods = nf_core.modules.PipelineModules(ctx.obj["modules_repo_obj"])
+    mods = nf_core.modules.PipelineModules()
+    mods.modules_repo = ctx.obj["modules_repo_obj"]
     mods.update(tool)
 
 
@@ -343,7 +348,8 @@ def remove(ctx, tool):
     """
     Remove a software wrapper from a pipeline.
     """
-    mods = nf_core.modules.PipelineModules(ctx.obj["modules_repo_obj"])
+    mods = nf_core.modules.PipelineModules()
+    mods.modules_repo = ctx.obj["modules_repo_obj"]
     mods.remove(tool)
 
 
@@ -360,7 +366,8 @@ def check(ctx):
     Use by the lint tests and automated CI to check that centralised
     software wrapper code is only modified in the central repository.
     """
-    mods = nf_core.modules.PipelineModules(ctx.obj["modules_repo_obj"])
+    mods = nf_core.modules.PipelineModules()
+    mods.modules_repo = ctx.obj["modules_repo_obj"]
     mods.check_modules()
 
 
diff --git a/nf_core/modules.py b/nf_core/modules.py
index d8d83b6d31..ecbe6bb5d8 100644
--- a/nf_core/modules.py
+++ b/nf_core/modules.py
@@ -27,12 +27,12 @@ def __init__(self, repo="nf-core/modules", branch="master"):
 
 
 class PipelineModules(object):
-    def __init__(self, modules_repo_obj):
+    def __init__(self):
         """
         Initialise the PipelineModules object
         """
-        self.modules_repo = modules_repo_obj
-        self.pipeline_dir = os.getcwd()
+        self.modules_repo = None
+        self.pipeline_dir = None
         self.modules_file_tree = {}
         self.modules_current_hash = None
         self.modules_avail_module_names = []
@@ -55,6 +55,16 @@ def list_modules(self):
 
     def install(self, module):
 
+        # Check that we were given a pipeline
+        if self.pipeline_dir is None or not os.path.exists(self.pipeline_dir):
+            logging.error("Could not find pipeline: {}".format(self.pipeline_dir))
+            return False
+        main_nf = os.path.join(self.pipeline_dir, "main.nf")
+        nf_config = os.path.join(self.pipeline_dir, "nextflow.config")
+        if not os.path.exists(main_nf) and not os.path.exists(nf_config):
+            logging.error("Could not find a main.nf or nextfow.config file in: {}".format(self.pipeline_dir))
+            return False
+
         # Get the available modules
         self.get_modules_file_tree()
 
@@ -132,7 +142,7 @@ def get_module_file_urls(self, module):
 
         Takes the name of a module and iterates over the GitHub repo file tree.
         Loops over items that are prefixed with the path 'software/<module_name>' and ignores
-        anything that's not a blob.
+        anything that's not a blob. Also ignores the test/ subfolder.
 
         Returns a dictionary with keys as filenames and values as GitHub API URIs.
         These can be used to then download file contents.
@@ -150,8 +160,13 @@ def get_module_file_urls(self, module):
         """
         results = {}
         for f in self.modules_file_tree:
-            if f["path"].startswith("software/{}".format(module)) and f["type"] == "blob":
-                results[f["path"]] = f["url"]
+            if not f["path"].startswith("software/{}".format(module)):
+                continue
+            if f["type"] != "blob":
+                continue
+            if "/test/" in f["path"]:
+                continue
+            results[f["path"]] = f["url"]
         return results
 
     def download_gh_file(self, dl_filename, api_url):

From 1c22055cd1a5944147e658858f332b1de1a6b4f9 Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Mon, 13 Jul 2020 11:05:10 +0200
Subject: [PATCH 13/14] Modules: Add some tests

---
 nf_core/__main__.py   |  2 +-
 nf_core/modules.py    | 10 +++++---
 tests/test_modules.py | 60 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 67 insertions(+), 5 deletions(-)
 create mode 100644 tests/test_modules.py

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index 7275fd6e5a..a64b973acf 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -302,7 +302,7 @@ def list(ctx):
     """
     mods = nf_core.modules.PipelineModules()
     mods.modules_repo = ctx.obj["modules_repo_obj"]
-    mods.list_modules()
+    print(mods.list_modules())
 
 
 @modules.command(help_priority=2)
diff --git a/nf_core/modules.py b/nf_core/modules.py
index ecbe6bb5d8..df8c91b048 100644
--- a/nf_core/modules.py
+++ b/nf_core/modules.py
@@ -31,7 +31,7 @@ def __init__(self):
         """
         Initialise the PipelineModules object
         """
-        self.modules_repo = None
+        self.modules_repo = ModulesRepo()
         self.pipeline_dir = None
         self.modules_file_tree = {}
         self.modules_current_hash = None
@@ -43,15 +43,17 @@ def list_modules(self):
         and print as list to stdout
         """
         self.get_modules_file_tree()
+        return_str = ""
 
         if len(self.modules_avail_module_names) > 0:
-            logging.info("Software available from {} ({}):\n".format(self.modules_repo.name, self.modules_repo.branch))
+            logging.info("Modules available from {} ({}):\n".format(self.modules_repo.name, self.modules_repo.branch))
             # Print results to stdout
-            print("\n".join(self.modules_avail_module_names))
+            return_str += "\n".join(self.modules_avail_module_names)
         else:
             logging.info(
-                "No available software found in {} ({}):\n".format(self.modules_repo.name, self.modules_repo.branch)
+                "No available modules found in {} ({}):\n".format(self.modules_repo.name, self.modules_repo.branch)
             )
+        return return_str
 
     def install(self, module):
 
diff --git a/tests/test_modules.py b/tests/test_modules.py
new file mode 100644
index 0000000000..7643c70fc5
--- /dev/null
+++ b/tests/test_modules.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+""" Tests covering the modules commands
+"""
+
+import nf_core.modules
+
+import mock
+import os
+import shutil
+import tempfile
+import unittest
+
+
+class TestModules(unittest.TestCase):
+    """Class for modules tests"""
+
+    def setUp(self):
+        """ Create a new PipelineSchema and Launch objects """
+        # Set up the schema
+        root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+        self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template", "{{cookiecutter.name_noslash}}")
+        self.pipeline_dir = os.path.join(tempfile.mkdtemp(), "mypipeline")
+        shutil.copytree(self.template_dir, self.pipeline_dir)
+        self.mods = nf_core.modules.PipelineModules()
+        self.mods.pipeline_dir = self.pipeline_dir
+
+    def test_modulesrepo_class(self):
+        """ Initialise a modules repo object """
+        modrepo = nf_core.modules.ModulesRepo()
+        assert modrepo.name == "nf-core/modules"
+        assert modrepo.branch == "master"
+
+    def test_modules_list(self):
+        """ Test listing available modules """
+        self.mods.pipeline_dir = None
+        listed_mods = self.mods.list_modules()
+        assert "fastqc" in listed_mods
+
+    def test_modules_install_nopipeline(self):
+        """ Test installing a module - no pipeline given """
+        self.mods.pipeline_dir = None
+        assert self.mods.install("foo") is False
+
+    def test_modules_install_emptypipeline(self):
+        """ Test installing a module - empty dir given """
+        self.mods.pipeline_dir = tempfile.mkdtemp()
+        assert self.mods.install("foo") is False
+
+    def test_modules_install_nomodule(self):
+        """ Test installing a module - unrecognised module given """
+        assert self.mods.install("foo") is False
+
+    def test_modules_install_fastqc(self):
+        """ Test installing a module - FastQC """
+        assert self.mods.install("fastqc") is not False
+
+    def test_modules_install_fastqc_twice(self):
+        """ Test installing a module - FastQC already there """
+        self.mods.install("fastqc")
+        assert self.mods.install("fastqc") is False

From b12a82e6103141970b70cffda3e1399c2acdefa8 Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil.ewels@scilifelab.se>
Date: Mon, 13 Jul 2020 11:09:26 +0200
Subject: [PATCH 14/14] Update modules subcommands to take a pipeline dir path

---
 nf_core/__main__.py | 12 ++++++++----
 nf_core/modules.py  |  2 +-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index a64b973acf..aacc439950 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -324,9 +324,10 @@ def install(ctx, pipeline_dir, tool):
 
 @modules.command(help_priority=3)
 @click.pass_context
+@click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="<pipeline directory>")
 @click.argument("tool", type=str, metavar="<tool name>")
-# --force - overwrite files even if no update found
-def update(ctx, tool):
+@click.option("-f", "--force", is_flag=True, default=False, help="Force overwrite of files")
+def update(ctx, tool, pipeline_dir, force):
     """
     Update one or all software wrapper modules.
 
@@ -338,18 +339,21 @@ def update(ctx, tool):
     """
     mods = nf_core.modules.PipelineModules()
     mods.modules_repo = ctx.obj["modules_repo_obj"]
-    mods.update(tool)
+    mods.pipeline_dir = pipeline_dir
+    mods.update(tool, force=force)
 
 
 @modules.command(help_priority=4)
 @click.pass_context
+@click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="<pipeline directory>")
 @click.argument("tool", type=str, required=True, metavar="<tool name>")
-def remove(ctx, tool):
+def remove(ctx, pipeline_dir, tool):
     """
     Remove a software wrapper from a pipeline.
     """
     mods = nf_core.modules.PipelineModules()
     mods.modules_repo = ctx.obj["modules_repo_obj"]
+    mods.pipeline_dir = pipeline_dir
     mods.remove(tool)
 
 
diff --git a/nf_core/modules.py b/nf_core/modules.py
index df8c91b048..1838685827 100644
--- a/nf_core/modules.py
+++ b/nf_core/modules.py
@@ -91,7 +91,7 @@ def install(self, module):
             dl_filename = os.path.join(self.pipeline_dir, "modules", filename)
             self.download_gh_file(dl_filename, api_url)
 
-    def update(self, module):
+    def update(self, module, force=False):
         logging.error("This command is not yet implemented")
         pass