From 390c03a57e5e3d9a5c68bdf93988d32340748d4a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 20 Dec 2024 14:23:56 +0100 Subject: [PATCH 01/44] bump to 3.2.0dev --- .gitpod.yml | 2 +- CHANGELOG.md | 14 ++++++++++++++ setup.py | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index db31d01bed..d5948695bf 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:latest +image: nfcore/gitpod:dev tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 926535077d..1f8a0386d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # nf-core/tools: Changelog +## v3.2.0dev + +### Template + +### Linting + +### Modules + +### Subworkflows + +### General + +### Version updates + ## [v3.1.1 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.1) - [2024-12-20] ### Template diff --git a/setup.py b/setup.py index 5617520e91..4a142c20fd 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "3.1.1" +version = "3.2.0dev" with open("README.md") as f: readme = f.read() From 40fa26ec8721a6046cfa7491b3e8742869622b9f Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 20 Dec 2024 13:25:22 +0000 Subject: [PATCH 02/44] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f8a0386d3..5f2f92f5fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ ### General +- bump to 3.2.0dev ([#3370](https://github.com/nf-core/tools/pull/3370)) + ### Version updates ## [v3.1.1 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.1) - [2024-12-20] From 20de879078501417c6fe5c69a065e35da8b3622f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 20 Dec 2024 15:01:11 +0100 Subject: [PATCH 03/44] fix trailing spaces in conf/test.config --- nf_core/pipeline-template/conf/test.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index bea6f670d0..54543cae1f 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -27,7 +27,7 @@ params { // TODO nf-core: Give any required params for the test so that command line flags are not needed input = params.pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' - {% if igenomes -%} + {%- if igenomes -%} // Genome references genome = 'R64-1-1' {%- endif %} From 3cb8775230583557698397bb07eeb69d8f73b2a5 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 20 Dec 2024 15:02:50 +0100 Subject: [PATCH 04/44] run pre-comit when rendering template for pipelines sync --- nf_core/pipelines/create/create.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/pipelines/create/create.py b/nf_core/pipelines/create/create.py index 4f90ca17f9..439cd5a359 100644 --- a/nf_core/pipelines/create/create.py +++ b/nf_core/pipelines/create/create.py @@ -386,6 +386,9 @@ def render_template(self) -> None: yaml.dump(config_yml.model_dump(exclude_none=True), fh, Dumper=custom_yaml_dumper()) log.debug(f"Dumping pipeline template yml to pipeline config file '{config_fn.name}'") + # Run prettier on files for pipelines sync + run_prettier_on_file([str(f) for f in self.outdir.glob("**/*")]) + def fix_linting(self): """ Updates the .nf-core.yml with linting configurations From 6b43881c39e63f8ac5be9319efd2a849c7893982 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 20 Dec 2024 15:19:15 +0100 Subject: [PATCH 05/44] add newline --- nf_core/pipeline-template/conf/test.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index 54543cae1f..ebe720f295 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -28,6 +28,7 @@ params { input = params.pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' {%- if igenomes -%} + // Genome references genome = 'R64-1-1' {%- endif %} From 4bd4d239744cd725be63c1a93aa3ab2f0e7a78e8 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 20 Dec 2024 15:21:14 +0100 Subject: [PATCH 06/44] bump to 3.1.2dev --- .gitpod.yml | 2 +- CHANGELOG.md | 16 ++++++++++++++++ setup.py | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index db31d01bed..d5948695bf 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:latest +image: nfcore/gitpod:dev tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 926535077d..aa465b85b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # nf-core/tools: Changelog +## v3.1.2dev + +### Template + +### Linting + +### Modules + +### Subworkflows + +### General + +- run pre-comit when rendering template for pipelines sync + +### Version updates + ## [v3.1.1 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.1) - [2024-12-20] ### Template diff --git a/setup.py b/setup.py index 5617520e91..fb1621adfc 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "3.1.1" +version = "3.1.2" with open("README.md") as f: readme = f.read() From bf3224c14d1c44796e322f6a46da5fa88d5a527a Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 20 Dec 2024 22:35:15 +0000 Subject: [PATCH 07/44] Allow downloading a specific commit ID --- nf_core/pipelines/download.py | 15 +++++++++++---- nf_core/utils.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/nf_core/pipelines/download.py b/nf_core/pipelines/download.py index 11adebce2c..e46c43ad6b 100644 --- a/nf_core/pipelines/download.py +++ b/nf_core/pipelines/download.py @@ -374,22 +374,27 @@ def prompt_revision(self) -> None: raise AssertionError(f"No revisions of {self.pipeline} available for download.") def get_revision_hash(self): - """Find specified revision / branch hash""" + """Find specified revision / branch / commit hash""" for revision in self.revision: # revision is a list of strings, but may be of length 1 # Branch if revision in self.wf_branches.keys(): self.wf_sha = {**self.wf_sha, revision: self.wf_branches[revision]} - # Revision else: + # Revision for r in self.wf_revisions: if r["tag_name"] == revision: self.wf_sha = {**self.wf_sha, revision: r["tag_sha"]} break - # Can't find the revisions or branch - throw an error else: + # Commit - full or short hash + if commit_id := nf_core.utils.get_repo_commit(self.pipeline, revision): + self.wf_sha = {**self.wf_sha, revision: commit_id} + continue + + # Can't find the revisions or branch - throw an error log.info( "Available {} revisions: '{}'".format( self.pipeline, @@ -397,7 +402,9 @@ def get_revision_hash(self): ) ) log.info("Available {} branches: '{}'".format(self.pipeline, "', '".join(self.wf_branches.keys()))) - raise AssertionError(f"Not able to find revision / branch '{revision}' for {self.pipeline}") + raise AssertionError( + f"Not able to find revision / branch / commit '{revision}' for {self.pipeline}" + ) # Set the outdir if not self.outdir: diff --git a/nf_core/utils.py b/nf_core/utils.py index e2b61329cc..6734ab50c0 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1096,6 +1096,16 @@ def get_repo_releases_branches(pipeline, wfs): return pipeline, wf_releases, wf_branches +def get_repo_commit(pipeline, commit_id): + commit_response = gh_api.get( + f"https://api.github.com/repos/{pipeline}/commits/{commit_id}", headers={"Accept": "application/vnd.github.sha"} + ) + if commit_response.status_code == 200: + return commit_response.text + else: + return None + + CONFIG_PATHS = [".nf-core.yml", ".nf-core.yaml"] DEPRECATED_CONFIG_PATHS = [".nf-core-lint.yml", ".nf-core-lint.yaml"] From 9dcb4a0a3adcf89264efda6b9e43592672dd860b Mon Sep 17 00:00:00 2001 From: fellen31 Date: Thu, 4 Apr 2024 07:49:21 +0200 Subject: [PATCH 08/44] Added tests --- tests/pipelines/test_download.py | 42 ++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/pipelines/test_download.py b/tests/pipelines/test_download.py index d1e2c41a68..c68d2ecc58 100644 --- a/tests/pipelines/test_download.py +++ b/tests/pipelines/test_download.py @@ -81,6 +81,48 @@ def test_get_release_hash_branch(self): == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" ) + def test_get_release_hash_long_commit(self): + wfs = nf_core.pipelines.list.Workflows() + wfs.get_remote_workflows() + # Exoseq pipeline is archived, so `dev` branch should be stable + pipeline = "exoseq" + + download_obj = DownloadWorkflow(pipeline="exoseq", revision="819cbac792b76cf66c840b567ed0ee9a2f620db7") + ( + download_obj.pipeline, + download_obj.wf_revisions, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) + download_obj.get_revision_hash() + print(download_obj) + assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert download_obj.outdir == "nf-core-exoseq_819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert ( + download_obj.wf_download_url[download_obj.revision[0]] + == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" + ) + + def test_get_release_hash_short_commit(self): + wfs = nf_core.pipelines.list.Workflows() + wfs.get_remote_workflows() + # Exoseq pipeline is archived, so `dev` branch should be stable + pipeline = "exoseq" + + download_obj = DownloadWorkflow(pipeline="exoseq", revision="819cbac") + ( + download_obj.pipeline, + download_obj.wf_revisions, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) + download_obj.get_revision_hash() + print(download_obj) + assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert download_obj.outdir == "nf-core-exoseq_819cbac" + assert ( + download_obj.wf_download_url[download_obj.revision[0]] + == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" + ) + def test_get_release_hash_non_existent_release(self): wfs = nf_core.pipelines.list.Workflows() wfs.get_remote_workflows() From 7b931f1fac9dec6d42a75cb0fa97c9263e7eae91 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 20 Dec 2024 23:04:45 +0000 Subject: [PATCH 09/44] Added more tests --- tests/test_utils.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index b13c8eb37d..0418c605cc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -169,6 +169,26 @@ def test_get_repo_releases_branches_not_exists_slash(self): with pytest.raises(AssertionError): nf_core.utils.get_repo_releases_branches("made-up/pipeline", wfs) + def test_get_repo_commit(self): + # The input can be a commit in standard long/short form, but also any length as long as it can be uniquely resolved + assert ( + nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3b95aaf01d98391a62a10a3990c0a4de395") + == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + ) + assert ( + nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3b95aaf01d") + == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + ) + assert ( + nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3b") == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + ) + assert ( + nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3") == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + ) + assert nf_core.utils.get_repo_commit("nf-core/methylseq", "xyz") is None + assert nf_core.utils.get_repo_commit("made_up_pipeline", "") is None + assert nf_core.utils.get_repo_commit("made-up/pipeline", "") is None + def test_validate_file_md5(self): # MD5(test) = d8e8fca2dc0f896fd7cb4cb0031ba249 test_file = TEST_DATA_DIR / "test.txt" From c3563374e11d628c50c0cb9d1b3272d02c58b745 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 20 Dec 2024 23:29:09 +0000 Subject: [PATCH 10/44] Updated the changelog --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 926535077d..95b7d3783b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # nf-core/tools: Changelog +## v3.2.0dev + +### Template + +### Download + +- Allow `nf-core pipelines download -r` to download commits ([#3374](https://github.com/nf-core/tools/pull/3374)) + +### Linting + +### Modules + +### Subworkflows + +### General + +### Version updates + ## [v3.1.1 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.1) - [2024-12-20] ### Template From b0b9559594f3cd2d55670c2b838f2b14a6ac4731 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 20 Dec 2024 23:34:44 +0000 Subject: [PATCH 11/44] Added some documentation --- nf_core/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nf_core/utils.py b/nf_core/utils.py index 6734ab50c0..2ac0943c59 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1097,6 +1097,16 @@ def get_repo_releases_branches(pipeline, wfs): def get_repo_commit(pipeline, commit_id): + """Check if the repo contains the requested commit_id, and expand it to long form if necessary. + + Args: + pipeline (str): GitHub repo username/repo + commit_id: The requested commit ID (SHA). It can be in standard long/short form, or any length. + + Returns: + commit_id: String or None + """ + commit_response = gh_api.get( f"https://api.github.com/repos/{pipeline}/commits/{commit_id}", headers={"Accept": "application/vnd.github.sha"} ) From 132d09c7f0098a0d9b47ce2bc91b04d87aae49f3 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 1 Jan 2025 03:24:57 +0000 Subject: [PATCH 12/44] Update pre-commit hook pre-commit/mirrors-mypy to v1.14.1 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 68a6fa3ed7..24152cf748 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: alias: ec - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.13.0" + rev: "v1.14.1" hooks: - id: mypy additional_dependencies: From 6fe8f289e74b215d41a83c498f5f6a16ac8dbe74 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 4 Jan 2025 12:37:57 +0000 Subject: [PATCH 13/44] Update pre-commit hook astral-sh/ruff-pre-commit to v0.8.6 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 68a6fa3ed7..69d095c27b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.2 + rev: v0.8.6 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From a335212e8cd0a65f07b9deed4ea1a31ae3f35648 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Jan 2025 15:01:56 +0100 Subject: [PATCH 14/44] only append module name if it is a dir and contains main.nf --- nf_core/modules/modules_utils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/modules_utils.py b/nf_core/modules/modules_utils.py index ecfe5f24ee..0f42d1bcea 100644 --- a/nf_core/modules/modules_utils.py +++ b/nf_core/modules/modules_utils.py @@ -65,19 +65,20 @@ def get_installed_modules(directory: Path, repo_type="modules") -> Tuple[List[st local_modules = sorted([x for x in local_modules if x.endswith(".nf")]) # Get nf-core modules - if os.path.exists(nfcore_modules_dir): - for m in sorted([m for m in os.listdir(nfcore_modules_dir) if not m == "lib"]): - if not os.path.isdir(os.path.join(nfcore_modules_dir, m)): + if nfcore_modules_dir.exists(): + for m in sorted([m for m in nfcore_modules_dir.iterdir() if not m == "lib"]): + if not m.is_dir(): raise ModuleExceptionError( f"File found in '{nfcore_modules_dir}': '{m}'! This directory should only contain module directories." ) - m_content = os.listdir(os.path.join(nfcore_modules_dir, m)) + m_content = [d.name for d in m.iterdir()] # Not a module, but contains sub-modules if "main.nf" not in m_content: for tool in m_content: - nfcore_modules_names.append(os.path.join(m, tool)) + if (m / tool).is_dir() and "main.nf" in [d.name for d in (m / tool).iterdir()]: + nfcore_modules_names.append(str(Path(m.name, tool))) else: - nfcore_modules_names.append(m) + nfcore_modules_names.append(m.name) # Make full (relative) file paths and create NFCoreComponent objects if local_modules_dir: From 983dcedfb1414f135049cb8460e8897fa15800b3 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Jan 2025 15:07:17 +0100 Subject: [PATCH 15/44] update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f2f92f5fe..1b006c98f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,12 @@ ### Modules +- Fix bump-versions: only append module name if it is a dir and contains main.nf ([#3384](https://github.com/nf-core/tools/pull/3384)) + ### Subworkflows ### General -- bump to 3.2.0dev ([#3370](https://github.com/nf-core/tools/pull/3370)) - ### Version updates ## [v3.1.1 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.1) - [2024-12-20] From 069fce2c0a181c3731d21a8ae48838e248ca2c81 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Jan 2025 15:43:06 +0100 Subject: [PATCH 16/44] add tests for get_installed_modules() utils function --- tests/modules/test_modules_utils.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/modules/test_modules_utils.py diff --git a/tests/modules/test_modules_utils.py b/tests/modules/test_modules_utils.py new file mode 100644 index 0000000000..763725337b --- /dev/null +++ b/tests/modules/test_modules_utils.py @@ -0,0 +1,20 @@ +import nf_core.modules.modules_utils + +from ..test_modules import TestModules + + +class TestModulesUtils(TestModules): + def test_get_installed_modules(self): + """Test getting installed modules""" + _, nfcore_modules = nf_core.modules.modules_utils.get_installed_modules(self.nfcore_modules) + assert len(nfcore_modules) == 1 + assert nfcore_modules[0].component_name == "bpipe/test" + + def test_get_installed_modules_with_files(self): + """Test getting installed modules. When a module contains a file in its directory, it shouldn't be picked up as a tool/subtool""" + # Create a file in the module directory + with open(self.nfcore_modules / "modules" / "nf-core" / "bpipe" / "test_file.txt", "w") as fh: + fh.write("test") + + _, nfcore_modules = nf_core.modules.modules_utils.get_installed_modules(self.nfcore_modules) + assert len(nfcore_modules) == 1 From 19070e53c03fbb5eb1f90edd0bd2529264d4368c Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 7 Jan 2025 16:22:32 +0100 Subject: [PATCH 17/44] remove type hint to make mypy pass --- nf_core/pipelines/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/download.py b/nf_core/pipelines/download.py index 11adebce2c..31dd4b06bd 100644 --- a/nf_core/pipelines/download.py +++ b/nf_core/pipelines/download.py @@ -1536,7 +1536,7 @@ def singularity_pull_image( progress.remove_task(task) - def compress_download(self) -> None: + def compress_download(self): """Take the downloaded files and make a compressed .tar.gz archive.""" log.debug(f"Creating archive: {self.output_filename}") From 011d15ab61013092d6bb14dd324fa46bf8a9b937 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2025 16:10:17 +0000 Subject: [PATCH 18/44] Update python:3.12-slim Docker digest to 10f3aaa (#3381) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f2141145b8..f5a6796a27 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:2b0079146a74e23bf4ae8f6a28e1b484c6292f6fb904cbb51825b4a19812fcd8 +FROM python:3.12-slim@sha256:10f3aaab98db50cba827d3b33a91f39dc9ec2d02ca9b85cbc5008220d07b17f3 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" From 4a74d5b844999b772d4d5ca705d9fa34619d3636 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Jan 2025 11:14:08 +0100 Subject: [PATCH 19/44] remove required from oneOf, allOf and anyOf instances in a schema --- nf_core/pipelines/schema.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index a08dd0a2d0..b9e2a049fa 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -330,6 +330,33 @@ def validate_default_params(self): for group_key, group in schema_no_required.get(self.defs_notation, {}).items(): if "required" in group: schema_no_required[self.defs_notation][group_key].pop("required") + if "allOf" in group: + for all_of in group["allOf"]: + if "required" in all_of: + schema_no_required[self.defs_notation][group_key]["allOf"].pop("required") + schema_no_required[self.defs_notation][group_key]["allOf"] = [ + all_of for all_of in group["allOf"] if all_of + ] + if not group["allOf"]: + schema_no_required[self.defs_notation][group_key].pop("allOf") + if "anyOf" in group: + for any_of in group["anyOf"]: + if "required" in any_of: + schema_no_required[self.defs_notation][group_key]["anyOf"].pop("required") + schema_no_required[self.defs_notation][group_key]["anyOf"] = [ + any_of for any_of in group["anyOf"] if any_of + ] + if not group["anyOf"]: + schema_no_required[self.defs_notation][group_key].pop("anyOf") + if "oneOf" in group: + for i, one_of in enumerate(group["oneOf"]): + if "required" in one_of: + schema_no_required[self.defs_notation][group_key]["oneOf"][i].pop("required") + schema_no_required[self.defs_notation][group_key]["oneOf"] = [ + one_of for one_of in group["oneOf"] if one_of + ] + if not group["oneOf"]: + schema_no_required[self.defs_notation][group_key].pop("oneOf") jsonschema.validate(self.schema_defaults, schema_no_required) except jsonschema.exceptions.ValidationError as e: raise AssertionError(f"Default parameters are invalid: {e.message}") From 5782edfae92a7f2460c703d801eafd003c8cf737 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Jan 2025 11:14:37 +0100 Subject: [PATCH 20/44] print complete error message from schema validation on debug --- nf_core/pipelines/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index b9e2a049fa..5718db4ef9 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -359,6 +359,7 @@ def validate_default_params(self): schema_no_required[self.defs_notation][group_key].pop("oneOf") jsonschema.validate(self.schema_defaults, schema_no_required) except jsonschema.exceptions.ValidationError as e: + log.debug(f"Complete error message:\n{e}") raise AssertionError(f"Default parameters are invalid: {e.message}") for param, default in self.schema_defaults.items(): if default in ("null", "", None, "None") or default is False: From 958f58ec165cb5ac34890a63643b5c47151cf1a8 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 8 Jan 2025 10:26:52 +0000 Subject: [PATCH 21/44] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b006c98f3..3f9ba0fea9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ ### General +- Parameters schema validation: allow oneOf, anyOf and allOf with `required` ([#3386](https://github.com/nf-core/tools/pull/3386)) + ### Version updates ## [v3.1.1 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.1) - [2024-12-20] From 51fba5aa9ef0a7a9e4d2982aa9dcafdb0b9790ef Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Jan 2025 15:04:02 +0100 Subject: [PATCH 22/44] reduce code --- nf_core/pipelines/schema.py | 37 ++++++++++--------------------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 5718db4ef9..68e1eab933 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -330,33 +330,16 @@ def validate_default_params(self): for group_key, group in schema_no_required.get(self.defs_notation, {}).items(): if "required" in group: schema_no_required[self.defs_notation][group_key].pop("required") - if "allOf" in group: - for all_of in group["allOf"]: - if "required" in all_of: - schema_no_required[self.defs_notation][group_key]["allOf"].pop("required") - schema_no_required[self.defs_notation][group_key]["allOf"] = [ - all_of for all_of in group["allOf"] if all_of - ] - if not group["allOf"]: - schema_no_required[self.defs_notation][group_key].pop("allOf") - if "anyOf" in group: - for any_of in group["anyOf"]: - if "required" in any_of: - schema_no_required[self.defs_notation][group_key]["anyOf"].pop("required") - schema_no_required[self.defs_notation][group_key]["anyOf"] = [ - any_of for any_of in group["anyOf"] if any_of - ] - if not group["anyOf"]: - schema_no_required[self.defs_notation][group_key].pop("anyOf") - if "oneOf" in group: - for i, one_of in enumerate(group["oneOf"]): - if "required" in one_of: - schema_no_required[self.defs_notation][group_key]["oneOf"][i].pop("required") - schema_no_required[self.defs_notation][group_key]["oneOf"] = [ - one_of for one_of in group["oneOf"] if one_of - ] - if not group["oneOf"]: - schema_no_required[self.defs_notation][group_key].pop("oneOf") + for keyword in ["allOf", "anyOf", "oneOf"]: + if keyword in group: + for i, kw_content in enumerate(group[keyword]): + if "required" in kw_content: + schema_no_required[self.defs_notation][group_key][keyword][i].pop("required") + schema_no_required[self.defs_notation][group_key][keyword] = [ + kw_content for kw_content in group[keyword] if kw_content + ] + if not group[keyword]: + schema_no_required[self.defs_notation][group_key].pop(keyword) jsonschema.validate(self.schema_defaults, schema_no_required) except jsonschema.exceptions.ValidationError as e: log.debug(f"Complete error message:\n{e}") From 2f2950fd12a197f08bb30ae7fe0cd97b8b768a10 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Jan 2025 15:44:18 +0100 Subject: [PATCH 23/44] handle anyOf, allOf, oneOf in top level --- nf_core/pipelines/schema.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 68e1eab933..b425ec64ed 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -327,6 +327,16 @@ def validate_default_params(self): schema_no_required = copy.deepcopy(self.schema) if "required" in schema_no_required: schema_no_required.pop("required") + for keyword in ["allOf", "anyOf", "oneOf"]: + if keyword in schema_no_required: + for i, kw_content in enumerate(schema_no_required[keyword]): + if "required" in kw_content: + schema_no_required[keyword][i].pop("required") + schema_no_required[keyword] = [ + kw_content for kw_content in schema_no_required[keyword] if kw_content + ] + if not schema_no_required[keyword]: + schema_no_required.pop(keyword) for group_key, group in schema_no_required.get(self.defs_notation, {}).items(): if "required" in group: schema_no_required[self.defs_notation][group_key].pop("required") From b1c7a8c597de9ca131437f85118b7dccd65b60ed Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Jan 2025 15:44:48 +0100 Subject: [PATCH 24/44] add tests for validate_default_params function --- tests/pipelines/test_schema.py | 83 ++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/pipelines/test_schema.py b/tests/pipelines/test_schema.py index ab543d8b90..efc2798969 100644 --- a/tests/pipelines/test_schema.py +++ b/tests/pipelines/test_schema.py @@ -285,6 +285,89 @@ def test_remove_schema_notfound_configs_childschema(self): assert len(params_removed) == 1 assert "foo" in params_removed + def test_validate_defaults(self): + """Test validating default values""" + self.schema_obj.schema = { + "properties": {"foo": {"type": "string"}, "bar": {"type": "string"}}, + "required": ["foo"], + } + self.schema_obj.schema_defaults = {"foo": "foo", "bar": "bar"} + self.schema_obj.no_prompts = True + try: + self.schema_obj.validate_default_params() + except AssertionError: + self.fail("Error validating schema defaults") + + def test_validate_defaults_required(self): + """Test validating default values when required params don't have a default""" + self.schema_obj.schema = { + "properties": {"foo": {"type": "string"}, "bar": {"type": "string"}}, + "required": ["foo"], + } + self.schema_obj.schema_defaults = {} + self.schema_obj.no_prompts = True + try: + self.schema_obj.validate_default_params() + except AssertionError: + self.fail("Error validating schema defaults") + + def test_validate_defaults_required_inside_group(self): + """Test validating default values when required params don't have a default, inside a group""" + self.schema_obj.schema = { + "$defs": { + "subSchemaId": { + "properties": {"foo": {"type": "string"}, "bar": {"type": "string"}}, + "required": ["foo"], + }, + } + } + self.schema_obj.schema_defaults = {} + self.schema_obj.no_prompts = True + try: + self.schema_obj.validate_default_params() + except AssertionError: + self.fail("Error validating schema defaults") + + def test_validate_defaults_required_inside_group_with_anyof(self): + """Test validating default values when required params don't have a default, inside a group with anyOf""" + self.schema_obj.schema = { + "$defs": { + "subSchemaId": { + "anyOf": [{"required": ["foo"]}, {"required": ["bar"]}], + "properties": {"foo": {"type": "string"}, "bar": {"type": "string"}}, + }, + } + } + self.schema_obj.schema_defaults = {} + self.schema_obj.no_prompts = True + try: + self.schema_obj.validate_default_params() + except AssertionError: + self.fail("Error validating schema defaults") + + def test_validate_defaults_required_with_anyof(self): + """Test validating default values when required params don't have a default, with anyOf""" + self.schema_obj.schema = { + "properties": {"foo": {"type": "string"}, "bar": {"type": "string"}, "baz": {"type": "string"}}, + "anyOf": [{"required": ["foo"]}, {"required": ["bar"]}], + } + self.schema_obj.schema_defaults = {"baz": "baz"} + self.schema_obj.no_prompts = True + try: + self.schema_obj.validate_default_params() + except AssertionError: + self.fail("Error validating schema defaults") + + def test_validate_defaults_error(self): + """Test validating default raises an exception when a default is not valid""" + self.schema_obj.schema = { + "properties": {"foo": {"type": "string"}}, + } + self.schema_obj.schema_defaults = {"foo": 1} + self.schema_obj.no_prompts = True + with self.assertRaises(AssertionError): + self.schema_obj.validate_default_params() + def test_add_schema_found_configs(self): """Try adding a new parameter to the schema from the config""" self.schema_obj.pipeline_params = {"foo": "bar"} From 824fa8f2bf9510d5eba5d833f2d4086656c26bd9 Mon Sep 17 00:00:00 2001 From: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> Date: Wed, 8 Jan 2025 21:39:12 +0530 Subject: [PATCH 25/44] Update ci.yml --- nf_core/pipeline-template/.github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 9db393d9f0..f3a1073843 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -46,6 +46,8 @@ jobs: steps: - name: Check out pipeline code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 - name: Set up Nextflow uses: nf-core/setup-nextflow@v2 From 80e06dcbeb197c66683d66b6bb8ab663fd2a0a39 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 13 Jan 2025 12:12:00 +0100 Subject: [PATCH 26/44] do not run pre-commit if we are not on a git repo --- nf_core/pipelines/create/create.py | 1 + nf_core/pipelines/lint_utils.py | 44 ++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/nf_core/pipelines/create/create.py b/nf_core/pipelines/create/create.py index 439cd5a359..86ac022772 100644 --- a/nf_core/pipelines/create/create.py +++ b/nf_core/pipelines/create/create.py @@ -387,6 +387,7 @@ def render_template(self) -> None: log.debug(f"Dumping pipeline template yml to pipeline config file '{config_fn.name}'") # Run prettier on files for pipelines sync + log.debug("Running prettier on pipeline files") run_prettier_on_file([str(f) for f in self.outdir.glob("**/*")]) def fix_linting(self): diff --git a/nf_core/pipelines/lint_utils.py b/nf_core/pipelines/lint_utils.py index a6b98b1899..d41cf16b12 100644 --- a/nf_core/pipelines/lint_utils.py +++ b/nf_core/pipelines/lint_utils.py @@ -70,6 +70,15 @@ def print_fixes(lint_obj): ) +def check_git_repo() -> bool: + """Check if the current directory is a git repository.""" + try: + subprocess.check_output(["git", "rev-parse", "--is-inside-work-tree"]) + return True + except subprocess.CalledProcessError: + return False + + def run_prettier_on_file(file: Union[Path, str, List[str]]) -> None: """Run the pre-commit hook prettier on a file. @@ -80,6 +89,8 @@ def run_prettier_on_file(file: Union[Path, str, List[str]]) -> None: If Prettier is not installed, a warning is logged. """ + is_git = check_git_repo() + nf_core_pre_commit_config = Path(nf_core.__file__).parent / ".pre-commit-prettier-config.yaml" args = ["pre-commit", "run", "--config", str(nf_core_pre_commit_config), "prettier"] if isinstance(file, List): @@ -87,21 +98,24 @@ def run_prettier_on_file(file: Union[Path, str, List[str]]) -> None: else: args.extend(["--files", str(file)]) - try: - subprocess.run(args, capture_output=True, check=True) - log.debug(f"${subprocess.STDOUT}") - except subprocess.CalledProcessError as e: - if ": SyntaxError: " in e.stdout.decode(): - log.critical(f"Can't format {file} because it has a syntax error.\n{e.stdout.decode()}") - elif "files were modified by this hook" in e.stdout.decode(): - all_lines = [line for line in e.stdout.decode().split("\n")] - files = "\n".join(all_lines[3:]) - log.debug(f"The following files were modified by prettier:\n {files}") - else: - log.warning( - "There was an error running the prettier pre-commit hook.\n" - f"STDOUT: {e.stdout.decode()}\nSTDERR: {e.stderr.decode()}" - ) + if is_git: + try: + proc = subprocess.run(args, capture_output=True, check=True) + log.debug(f"{proc.stdout.decode()}") + except subprocess.CalledProcessError as e: + if ": SyntaxError: " in e.stdout.decode(): + log.critical(f"Can't format {file} because it has a syntax error.\n{e.stdout.decode()}") + elif "files were modified by this hook" in e.stdout.decode(): + all_lines = [line for line in e.stdout.decode().split("\n")] + files = "\n".join(all_lines[3:]) + log.debug(f"The following files were modified by prettier:\n {files}") + else: + log.warning( + "There was an error running the prettier pre-commit hook.\n" + f"STDOUT: {e.stdout.decode()}\nSTDERR: {e.stderr.decode()}" + ) + else: + log.debug("Not in a git repository, skipping pre-commit hook.") def dump_json_with_prettier(file_name, file_content): From a7aee9c9c9164df1cd9b0d5f685ca88f82e8a766 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 13 Jan 2025 16:18:17 +0000 Subject: [PATCH 27/44] Factored out the commit ID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- tests/pipelines/test_download.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/pipelines/test_download.py b/tests/pipelines/test_download.py index c68d2ecc58..1046634f36 100644 --- a/tests/pipelines/test_download.py +++ b/tests/pipelines/test_download.py @@ -86,8 +86,9 @@ def test_get_release_hash_long_commit(self): wfs.get_remote_workflows() # Exoseq pipeline is archived, so `dev` branch should be stable pipeline = "exoseq" + revision = "819cbac792b76cf66c840b567ed0ee9a2f620db7" - download_obj = DownloadWorkflow(pipeline="exoseq", revision="819cbac792b76cf66c840b567ed0ee9a2f620db7") + download_obj = DownloadWorkflow(pipeline=pipeline, revision=revision) ( download_obj.pipeline, download_obj.wf_revisions, @@ -95,11 +96,11 @@ def test_get_release_hash_long_commit(self): ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) download_obj.get_revision_hash() print(download_obj) - assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7" - assert download_obj.outdir == "nf-core-exoseq_819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert download_obj.wf_sha[download_obj.revision[0]] == revision + assert download_obj.outdir == f"nf-core-exoseq_{revision}" assert ( download_obj.wf_download_url[download_obj.revision[0]] - == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" + == f"https://github.com/nf-core/exoseq/archive/{revision}.zip" ) def test_get_release_hash_short_commit(self): From 1844f5587f19ef87db4b2c78d63f3112568537aa Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 13 Jan 2025 16:18:33 +0000 Subject: [PATCH 28/44] Not needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- tests/pipelines/test_download.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pipelines/test_download.py b/tests/pipelines/test_download.py index 1046634f36..608f524215 100644 --- a/tests/pipelines/test_download.py +++ b/tests/pipelines/test_download.py @@ -95,7 +95,6 @@ def test_get_release_hash_long_commit(self): download_obj.wf_branches, ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) download_obj.get_revision_hash() - print(download_obj) assert download_obj.wf_sha[download_obj.revision[0]] == revision assert download_obj.outdir == f"nf-core-exoseq_{revision}" assert ( From 5ae7d53a91e64e16b6dc103248a983d3fb2a2ffe Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 13 Jan 2025 16:22:29 +0000 Subject: [PATCH 29/44] Factored out the commit ID --- tests/test_utils.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0418c605cc..b7761253a3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -171,20 +171,11 @@ def test_get_repo_releases_branches_not_exists_slash(self): def test_get_repo_commit(self): # The input can be a commit in standard long/short form, but also any length as long as it can be uniquely resolved - assert ( - nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3b95aaf01d98391a62a10a3990c0a4de395") - == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" - ) - assert ( - nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3b95aaf01d") - == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" - ) - assert ( - nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3b") == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" - ) - assert ( - nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3") == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" - ) + revision = "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + assert nf_core.utils.get_repo_commit("nf-core/methylseq", revision) == revision + assert nf_core.utils.get_repo_commit("nf-core/methylseq", revision[:16]) == revision + assert nf_core.utils.get_repo_commit("nf-core/methylseq", revision[:7]) == revision + assert nf_core.utils.get_repo_commit("nf-core/methylseq", revision[:6]) == revision assert nf_core.utils.get_repo_commit("nf-core/methylseq", "xyz") is None assert nf_core.utils.get_repo_commit("made_up_pipeline", "") is None assert nf_core.utils.get_repo_commit("made-up/pipeline", "") is None From 62c3c2bc14a907a166f6cbff23b5ba1349b5f9cc Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 13 Jan 2025 16:24:41 +0000 Subject: [PATCH 30/44] Factored out the commit ID --- tests/pipelines/test_download.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/pipelines/test_download.py b/tests/pipelines/test_download.py index 608f524215..6db7392107 100644 --- a/tests/pipelines/test_download.py +++ b/tests/pipelines/test_download.py @@ -107,8 +107,10 @@ def test_get_release_hash_short_commit(self): wfs.get_remote_workflows() # Exoseq pipeline is archived, so `dev` branch should be stable pipeline = "exoseq" + revision = "819cbac792b76cf66c840b567ed0ee9a2f620db7" + short_rev = revision[:7] - download_obj = DownloadWorkflow(pipeline="exoseq", revision="819cbac") + download_obj = DownloadWorkflow(pipeline="exoseq", revision=short_rev) ( download_obj.pipeline, download_obj.wf_revisions, @@ -116,11 +118,11 @@ def test_get_release_hash_short_commit(self): ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) download_obj.get_revision_hash() print(download_obj) - assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7" - assert download_obj.outdir == "nf-core-exoseq_819cbac" + assert download_obj.wf_sha[download_obj.revision[0]] == revision + assert download_obj.outdir == f"nf-core-exoseq_{short_rev}" assert ( download_obj.wf_download_url[download_obj.revision[0]] - == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" + == f"https://github.com/nf-core/exoseq/archive/{revision}.zip" ) def test_get_release_hash_non_existent_release(self): From 16dd8d577e692e34d0e56ae40c641d457ad56004 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 8 Jan 2025 13:20:40 +0100 Subject: [PATCH 31/44] Reorganise download test to ensure setup and download are executed on the same runner. --- .../.github/workflows/download_pipeline.yml | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index f270dc5411..7fa591e559 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -35,7 +35,18 @@ jobs: REPOTITLE_LOWERCASE: ${{ steps.get_repo_properties.outputs.REPOTITLE_LOWERCASE }} REPO_BRANCH: ${{ steps.get_repo_properties.outputs.REPO_BRANCH }} steps: - - name: Install Nextflow{% endraw %} + - name: Get the repository name and current branch set as environment variable + id: get_repo_properties + run: | + echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT" + echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> "$GITHUB_OUTPUT" + echo "REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> "$GITHUB_OUTPUT{% endraw %}" + + download: + runs-on: ubuntu-latest + needs: configure + steps: + - name: Install Nextflow uses: nf-core/setup-nextflow@v2 - name: Disk space cleanup @@ -56,24 +67,13 @@ jobs: python -m pip install --upgrade pip pip install git+https://github.com/nf-core/tools.git@dev - - name: Get the repository name and current branch set as environment variable - id: get_repo_properties - run: | - echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT" - echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> "$GITHUB_OUTPUT" - echo "{% raw %}REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> "$GITHUB_OUTPUT" - - name: Make a cache directory for the container images run: | mkdir -p ./singularity_container_images - download: - runs-on: ubuntu-latest - needs: configure - steps: - name: Download the pipeline env: - NXF_SINGULARITY_CACHEDIR: ./singularity_container_images + NXF_SINGULARITY_CACHEDIR: ./singularity_container_images{% raw %} run: | nf-core pipelines download ${{ needs.configure.outputs.REPO_LOWERCASE }} \ --revision ${{ needs.configure.outputs.REPO_BRANCH }} \ From 2184cab8074d194662dd5607875824e747a36eb6 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 8 Jan 2025 16:52:26 +0100 Subject: [PATCH 32/44] Add explicit container cache inspection to action. --- .../.github/workflows/download_pipeline.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 7fa591e559..41deaaced2 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -87,6 +87,9 @@ jobs: - name: Inspect download run: tree ./${{ needs.configure.outputs.REPOTITLE_LOWERCASE }}{% endraw %}{% if test_config %}{% raw %} + - name: Inspect container images + run: tree ./singularity_container_images | tee ./container_initial + - name: Count the downloaded number of container images id: count_initial run: | @@ -123,7 +126,8 @@ jobs: final_count=${{ steps.count_afterwards.outputs.IMAGE_COUNT_AFTER }} difference=$((final_count - initial_count)) echo "$difference additional container images were \n downloaded at runtime . The pipeline has no support for offline runs!" - tree ./singularity_container_images + tree ./singularity_container_images > ./container_afterwards + diff ./container_initial ./container_afterwards exit 1 else echo "The pipeline can be downloaded successfully!" From 737b515d69e276f8ef83c343c5c13bbd0ee59938 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 8 Jan 2025 19:29:59 +0100 Subject: [PATCH 33/44] Changelog. --- CHANGELOG.md | 1 + .../.github/workflows/download_pipeline.yml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 704124baa9..35615729c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Download - Allow `nf-core pipelines download -r` to download commits ([#3374](https://github.com/nf-core/tools/pull/3374)) +- Fix faulty Download Test Action to ensure that setup and test run as one job and on the same runner ([#3389](https://github.com/nf-core/tools/pull/3389)) ### Linting diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 41deaaced2..977b4255dc 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -35,7 +35,7 @@ jobs: REPOTITLE_LOWERCASE: ${{ steps.get_repo_properties.outputs.REPOTITLE_LOWERCASE }} REPO_BRANCH: ${{ steps.get_repo_properties.outputs.REPO_BRANCH }} steps: - - name: Get the repository name and current branch set as environment variable + - name: Get the repository name and current branch id: get_repo_properties run: | echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT" @@ -85,10 +85,10 @@ jobs: --download-configuration 'yes' - name: Inspect download - run: tree ./${{ needs.configure.outputs.REPOTITLE_LOWERCASE }}{% endraw %}{% if test_config %}{% raw %} + run: tree ./${{ needs.configure.outputs.REPOTITLE_LOWERCASE }}{% endraw %} - name: Inspect container images - run: tree ./singularity_container_images | tee ./container_initial + run: tree ./singularity_container_images | tee ./container_initial{% if test_config %}{% raw %} - name: Count the downloaded number of container images id: count_initial From df41fc02e964eb12c8e7df027fd927060caa6b48 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 14 Jan 2025 10:00:09 +0100 Subject: [PATCH 34/44] fix sync GHA by removing quotes from parsed branch name --- .github/workflows/sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 7061294783..8e253bb3eb 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -97,7 +97,7 @@ jobs: run: | pushd nf-core/${{ matrix.pipeline }} - defaultBranch=$(grep -B5 -A5 "nextflowVersion" nextflow.config | grep "defaultBranch" | cut -d"=" -f2) + defaultBranch=$(grep -B5 -A5 "nextflowVersion" nextflow.config | grep "defaultBranch" | cut -d"=" -f2 | sed "s/'//g") if [ -z "$defaultBranch" ]; then defaultBranch="master" fi From ed1cbde1e918cd4a9b540e49867a0b1f0a327f6a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 14 Jan 2025 09:02:17 +0000 Subject: [PATCH 35/44] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35615729c0..5d0b9999f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Parameters schema validation: allow oneOf, anyOf and allOf with `required` ([#3386](https://github.com/nf-core/tools/pull/3386)) - Run pre-comit when rendering template for pipelines sync ([#3371](https://github.com/nf-core/tools/pull/3371)) +- Fix sync GHA by removing quotes from parsed branch name ([#3394](https://github.com/nf-core/tools/pull/3394)) ### Version updates From 4bdf77e28fc1cd29d5d7cc53eb960e09842ea1b0 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 14 Jan 2025 16:20:23 +0100 Subject: [PATCH 36/44] make rocrate no git info message a debug message to avoid seeing it every time we run nf-core pipelines lint --- nf_core/pipelines/rocrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/rocrate.py b/nf_core/pipelines/rocrate.py index f87cc7d8d2..b0f9611f1f 100644 --- a/nf_core/pipelines/rocrate.py +++ b/nf_core/pipelines/rocrate.py @@ -287,7 +287,7 @@ def add_main_authors(self, wf_file: rocrate.model.entity.Entity) -> None: try: git_contributors: Set[str] = set() if self.pipeline_obj.repo is None: - log.info("No git repository found. No git contributors will be added as authors.") + log.debug("No git repository found. No git contributors will be added as authors.") return commits_touching_path = list(self.pipeline_obj.repo.iter_commits(paths="main.nf")) From 225b0b7ed408db786fb411d65f576526c283277b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 14 Jan 2025 16:42:37 +0100 Subject: [PATCH 37/44] manifest.authors is not mandatory anymore --- nf_core/pipelines/lint/files_unchanged.py | 13 +++++++++++-- nf_core/pipelines/sync.py | 6 +++++- nf_core/utils.py | 4 +++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/nf_core/pipelines/lint/files_unchanged.py b/nf_core/pipelines/lint/files_unchanged.py index c1c3acd31f..4dcab3b657 100644 --- a/nf_core/pipelines/lint/files_unchanged.py +++ b/nf_core/pipelines/lint/files_unchanged.py @@ -1,6 +1,7 @@ import filecmp import logging import os +import re import shutil import tempfile from pathlib import Path @@ -68,7 +69,10 @@ def files_unchanged(self) -> Dict[str, Union[List[str], bool]]: could_fix: bool = False # Check that we have the minimum required config - required_pipeline_config = {"manifest.name", "manifest.description", "manifest.author"} + required_pipeline_config = { + "manifest.name", + "manifest.description", + } # TODO: add "manifest.contributors" when minimum nextflow version is >=24.10.0 missing_pipeline_config = required_pipeline_config.difference(self.nf_config) if missing_pipeline_config: return {"ignored": [f"Required pipeline config not found - {missing_pipeline_config}"]} @@ -117,10 +121,15 @@ def files_unchanged(self) -> Dict[str, Union[List[str], bool]]: tmp_dir.mkdir(parents=True) # Create a template.yaml file for the pipeline creation + if "manifest.author" in self.nf_config: + names = self.nf_config["manifest.author"].strip("\"'") + if "manifest.contributors" in self.nf_config: + contributors = self.nf_config["manifest.contributors"] + names = ", ".join(re.findall(r"name:'([^']+)'", contributors)) template_yaml = { "name": short_name, "description": self.nf_config["manifest.description"].strip("\"'"), - "author": self.nf_config["manifest.author"].strip("\"'"), + "author": names, "org": prefix, } diff --git a/nf_core/pipelines/sync.py b/nf_core/pipelines/sync.py index 781b4f5f00..14365da3f8 100644 --- a/nf_core/pipelines/sync.py +++ b/nf_core/pipelines/sync.py @@ -81,7 +81,11 @@ def __init__( self.made_changes = False self.make_pr = make_pr self.gh_pr_returned_data: Dict = {} - self.required_config_vars = ["manifest.name", "manifest.description", "manifest.version", "manifest.author"] + self.required_config_vars = [ + "manifest.name", + "manifest.description", + "manifest.version", + ] # TODO: add "manifest.contributors" when minimum nextflow version is >=24.10.0 self.force_pr = force_pr self.gh_username = gh_username diff --git a/nf_core/utils.py b/nf_core/utils.py index 2ac0943c59..27334d473c 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1352,8 +1352,10 @@ def load_tools_config(directory: Union[str, Path] = ".") -> Tuple[Optional[Path] contributors = wf_config["manifest.contributors"] names = re.findall(r"name:'([^']+)'", contributors) author_names = ", ".join(names) - else: + elif "manifest.author" in wf_config: author_names = wf_config["manifest.author"].strip("'\"") + else: + author_names = None if nf_core_yaml_config.template is None: # The .nf-core.yml file did not contain template information nf_core_yaml_config.template = NFCoreTemplateConfig( From 7641e5f84ca3768407fb1f4d1ffc0d0dfd07f815 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 14 Jan 2025 15:52:19 +0000 Subject: [PATCH 38/44] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35615729c0..0e001fed6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Parameters schema validation: allow oneOf, anyOf and allOf with `required` ([#3386](https://github.com/nf-core/tools/pull/3386)) - Run pre-comit when rendering template for pipelines sync ([#3371](https://github.com/nf-core/tools/pull/3371)) +- manifest.author is not required anymore ([#3397](https://github.com/nf-core/tools/pull/3397)) ### Version updates From cd72f9779ff82a23352384f986517ad7489e9f07 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 15 Jan 2025 14:16:05 +0100 Subject: [PATCH 39/44] update nf-schema version --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 3325af4e06..5bbe47d9d9 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -295,7 +295,7 @@ manifest { {% if nf_schema -%} // Nextflow plugins plugins { - id 'nf-schema@2.1.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.3.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { From e62a94bb8cfb581b6e50491d6a40e92d200b9e87 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 15 Jan 2025 14:17:20 +0100 Subject: [PATCH 40/44] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 319969dea1..2985dbd391 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Bump nf-schema to 2.3.0 ([#3401])(https://github.com/nf-core/tools/pull/3401) + ### Download - Allow `nf-core pipelines download -r` to download commits ([#3374](https://github.com/nf-core/tools/pull/3374)) From bd5c641848f7f1febed3e57a87d5ea9ac1319c6d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 20 Jan 2025 10:07:00 +0100 Subject: [PATCH 41/44] bump version to 3.1.2 --- .gitpod.yml | 2 +- CHANGELOG.md | 8 +------- setup.py | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index d5948695bf..db31d01bed 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:dev +image: nfcore/gitpod:latest tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 2985dbd391..c2ad2de40c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v3.1.2dev +## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-21] ### Template @@ -11,14 +11,10 @@ - Allow `nf-core pipelines download -r` to download commits ([#3374](https://github.com/nf-core/tools/pull/3374)) - Fix faulty Download Test Action to ensure that setup and test run as one job and on the same runner ([#3389](https://github.com/nf-core/tools/pull/3389)) -### Linting - ### Modules - Fix bump-versions: only append module name if it is a dir and contains main.nf ([#3384](https://github.com/nf-core/tools/pull/3384)) -### Subworkflows - ### General - Parameters schema validation: allow oneOf, anyOf and allOf with `required` ([#3386](https://github.com/nf-core/tools/pull/3386)) @@ -26,8 +22,6 @@ - Fix sync GHA by removing quotes from parsed branch name ([#3394](https://github.com/nf-core/tools/pull/3394)) - manifest.author is not required anymore ([#3397](https://github.com/nf-core/tools/pull/3397)) -### Version updates - ## [v3.1.1 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.1) - [2024-12-20] ### Template diff --git a/setup.py b/setup.py index ebf167a22c..fb1621adfc 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "3.1.2dev" +version = "3.1.2" with open("README.md") as f: readme = f.read() From 373ad61fa4a7cefe8badbb3ee9e361245a6cf4be Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 20 Jan 2025 10:16:22 +0100 Subject: [PATCH 42/44] remove jinja formatting which was deleting line breaks --- nf_core/pipeline-template/CITATIONS.md | 2 +- nf_core/pipeline-template/README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/CITATIONS.md b/nf_core/pipeline-template/CITATIONS.md index c355fd6129..16da9a4207 100644 --- a/nf_core/pipeline-template/CITATIONS.md +++ b/nf_core/pipeline-template/CITATIONS.md @@ -18,7 +18,7 @@ {%- endif %} -{%- if multiqc %}- [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/) +{% if multiqc %}- [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/) > Ewels P, Magnusson M, Lundin S, Käller M. MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics. 2016 Oct 1;32(19):3047-8. doi: 10.1093/bioinformatics/btw354. Epub 2016 Jun 16. PubMed PMID: 27312411; PubMed Central PMCID: PMC5039924. diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 4cd41de368..33015da914 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -7,7 +7,7 @@ -{%- else -%} +{% else -%} # {{ name }} @@ -54,7 +54,7 @@ ## Usage > [!NOTE] -> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. {%- if test_config %}Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data.{% endif %} +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. {% if test_config %}Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data.{% endif %} -{%- if citations %} +{% if citations %} An extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file. {%- endif %} From 53d0e33384c6d06832c9f278096a6e9456c1c95c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 20 Jan 2025 12:32:27 +0100 Subject: [PATCH 43/44] changelog updates --- CHANGELOG.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2ad2de40c..c400427dde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ ### Template -- Bump nf-schema to 2.3.0 ([#3401])(https://github.com/nf-core/tools/pull/3401) +- Bump nf-schema to `2.3.0` ([#3401](https://github.com/nf-core/tools/pull/3401)) +- Remove jinja formatting which was deleting line breaks ([#3405](https://github.com/nf-core/tools/pull/3405)) ### Download @@ -13,14 +14,14 @@ ### Modules -- Fix bump-versions: only append module name if it is a dir and contains main.nf ([#3384](https://github.com/nf-core/tools/pull/3384)) +- Fix bump-versions: only append module name if it is a dir and contains `main.nf` ([#3384](https://github.com/nf-core/tools/pull/3384)) ### General -- Parameters schema validation: allow oneOf, anyOf and allOf with `required` ([#3386](https://github.com/nf-core/tools/pull/3386)) +- `manifest.author` is not required anymore ([#3397](https://github.com/nf-core/tools/pull/3397)) +- Parameters schema validation: allow `oneOf`, `anyOf` and `allOf` with `required` ([#3386](https://github.com/nf-core/tools/pull/3386)) - Run pre-comit when rendering template for pipelines sync ([#3371](https://github.com/nf-core/tools/pull/3371)) - Fix sync GHA by removing quotes from parsed branch name ([#3394](https://github.com/nf-core/tools/pull/3394)) -- manifest.author is not required anymore ([#3397](https://github.com/nf-core/tools/pull/3397)) ## [v3.1.1 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.1) - [2024-12-20] From d113f247468343d4db800aff64571227b016ec42 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 20 Jan 2025 15:05:13 +0100 Subject: [PATCH 44/44] update release date [skip ci] --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c400427dde..1197a379b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-21] +## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] ### Template