diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b1941d81e..6d8e34e0b4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.4 + rev: v0.4.9 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix diff --git a/CHANGELOG.md b/CHANGELOG.md index 16262bd1c3..9da8c1ab24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,14 @@ ### Linting - Fix linting fail on nfcore_external_java_deps if nf_schema is used ([#2976](https://github.com/nf-core/tools/pull/2976)) +- Conda module linting: Include package name in log file ([#3014](https://github.com/nf-core/tools/pull/3014)) ### Download ### Components +- The `modules_nfcore` tag in the `main.nf.test` file of modules/subworkflows now displays the organization name in custom modules repositories ([#3005](https://github.com/nf-core/tools/pull/3005)) + ### General - Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.4 ([#2974](https://github.com/nf-core/tools/pull/2974)) @@ -21,12 +24,21 @@ - Update output of generation script for API docs to new structure ([#2988](https://github.com/nf-core/tools/pull/2988)) - Add no clobber and put bash options on their own line ([#2991](https://github.com/nf-core/tools/pull/2991)) - update minimal textual version and snapshots ([#2998](https://github.com/nf-core/tools/pull/2998)) +- return directory if base_dir is the root directory ([#3003](https://github.com/nf-core/tools/pull/3003)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.6 ([#3006](https://github.com/nf-core/tools/pull/3006)) +- Create: allow more special characters on the pipeline name for non-nf-core pipelines ([#3008](https://github.com/nf-core/tools/pull/3008)) +- README - absolute image paths ([#3013](https://github.com/nf-core/tools/pull/3013)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.7 ([#3015](https://github.com/nf-core/tools/pull/3015)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.8 ([#3017](https://github.com/nf-core/tools/pull/3017)) +- Update python:3.12-slim Docker digest to e3ae8cf ([#3020](https://github.com/nf-core/tools/pull/3020)) +- Update python:3.12-slim Docker digest to 2fba8e7 ([#3023](https://github.com/nf-core/tools/pull/3023)) ## [v2.14.1 - Tantalum Toad - Patch](https://github.com/nf-core/tools/releases/tag/2.14.1) - [2024-05-09] ### Template - Don't cache pip in `linting.yml` ([#2961](https://github.com/nf-core/tools/pull/2961)) +- Lint pipelines with the nf-core template version and post comment if it is outdated ([#2978](https://github.com/nf-core/tools/pull/2978)) ### General diff --git a/Dockerfile b/Dockerfile index fe4162b4fe..9ebc7e9853 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:afc139a0a640942491ec481ad8dda10f2c5b753f5c969393b12480155fe15a63 +FROM python:3.12-slim@sha256:2fba8e70a87bcc9f6edd20dda0a1d4adb32046d2acbca7361bc61da5a106a914 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" diff --git a/README.md b/README.md index 85b608bf77..58fb708a0d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

- - nf-core/tools + + nf-core/tools

diff --git a/nf_core/components/create.py b/nf_core/components/create.py index f2ffa3caf5..bce9bf1ff7 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -155,6 +155,10 @@ def create(self): if self.component_type == "modules": self._get_module_structure_components() + # Add a valid organization name for nf-test tags + not_alphabet = re.compile(r"[^a-zA-Z]") + self.org_alphabet = not_alphabet.sub("", self.org) + # Create component template with jinja2 self._render_template() log.info(f"Created component template: '{self.component_name}'") diff --git a/nf_core/module-template/tests/main.nf.test.j2 b/nf_core/module-template/tests/main.nf.test.j2 index 1f70df64b0..456c989f85 100644 --- a/nf_core/module-template/tests/main.nf.test.j2 +++ b/nf_core/module-template/tests/main.nf.test.j2 @@ -7,7 +7,7 @@ nextflow_process { process "{{ component_name_underscore|upper }}" tag "modules" - tag "modules_nfcore" + tag "modules_{{ org_alphabet }}" {%- if subtool %} tag "{{ component }}" {%- endif %} diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index fd4d81f7f2..81308ba5c5 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -367,14 +367,14 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): # response = _bioconda_package(bp) response = nf_core.utils.anaconda_package(bp) except LookupError: - self.warned.append(("bioconda_version", "Conda version not specified correctly", self.main_nf)) + self.warned.append(("bioconda_version", f"Conda version not specified correctly: {bp}", self.main_nf)) except ValueError: - self.failed.append(("bioconda_version", "Conda version not specified correctly", self.main_nf)) + self.failed.append(("bioconda_version", f"Conda version not specified correctly: {bp}", self.main_nf)) else: # Check that required version is available at all if bioconda_version not in response.get("versions"): self.failed.append( - ("bioconda_version", f"Conda package had unknown version: `{bioconda_version}`", self.main_nf) + ("bioconda_version", f"Conda package {bp} had unknown version: `{bioconda_version}`", self.main_nf) ) continue # No need to test for latest version, continue linting # Check version is latest available diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 694aee9c17..9301db81ee 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -4,6 +4,7 @@ import json import logging +import re from pathlib import Path import yaml @@ -169,7 +170,10 @@ def module_tests(_, module: NFCoreComponent): ) # Verify that tags are correct. main_nf_tags = module._get_main_nf_tags(module.nftest_main_nf) - required_tags = ["modules", "modules_nfcore", module.component_name] + not_alphabet = re.compile(r"[^a-zA-Z]") + org_alp = not_alphabet.sub("", module.org) + org_alphabet = org_alp if org_alp != "" else "nfcore" + required_tags = ["modules", f"modules_{org_alphabet}", module.component_name] if module.component_name.count("/") == 1: required_tags.append(module.component_name.split("/")[0]) chained_components_tags = module._get_included_components_in_chained_tests(module.nftest_main_nf) diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index b2cde075fe..229aa9f652 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -41,10 +41,16 @@ jobs: python-version: "3.12" architecture: "x64" + - name: read .nf-core.yml + uses: pietrobolcato/action-read-yaml@1.0.0 + id: read_yml + with: + config: ${{ github.workspace }}/.nf-core.yaml + - name: Install dependencies run: | python -m pip install --upgrade pip - pip install nf-core + pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }} - name: Run nf-core lint env: diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index ea408fd6f8..908dcea159 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3 + uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 # v6 with: workflow: linting.yml workflow_conclusion: completed diff --git a/nf_core/pipeline-template/.github/workflows/template_version_comment.yml b/nf_core/pipeline-template/.github/workflows/template_version_comment.yml new file mode 100644 index 0000000000..e21283309d --- /dev/null +++ b/nf_core/pipeline-template/.github/workflows/template_version_comment.yml @@ -0,0 +1,42 @@ +name: nf-core template version comment +# This workflow is triggered on PRs to check if the pipeline template version matches the latest nf-core version. +# It posts a comment to the PR, even if it comes from a fork. + +on: pull_request_target + +jobs: + template_version: + runs-on: ubuntu-latest + steps: + - name: Check out pipeline code + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 + + - name: Read template version from .nf-core.yml + uses: pietrobolcato/action-read-yaml@1.0.0 + id: read_yml + with: + config: ${{ github.workspace }}/.nf-core.yml + + - name: Install nf-core + run: | + python -m pip install --upgrade pip + pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }} + + - name: Check nf-core outdated + id: nf_core_outdated + run: pip list --outdated | grep nf-core + + - name: Post nf-core template version comment + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2 + if: | + ${{ steps.nf_core_outdated.outputs.stdout }} =~ 'nf-core' + with: + repo-token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} + allow-repeats: false + message: | + ## :warning: Newer version of the nf-core template is available. + + Your pipeline is using an old version of the nf-core template: ${{ steps.read_yml.outputs['nf_core_version'] }}. + Please update your pipeline to the latest version. + + For more documentation on how to update your pipeline, please see the [nf-core documentation](https://github.com/nf-core/tools?tab=readme-ov-file#sync-a-pipeline-with-the-template) and [Synchronisation documentation](https://nf-co.re/docs/contributing/sync). diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 4d1701bf76..2e6a56b001 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -187,13 +187,14 @@ profiles { test_full { includeConfig 'conf/test_full.config' } } -// Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile -// Will not be used unless Apptainer / Docker / Podman / Singularity are enabled +// Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile +// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity are enabled // Set to your registry if you have a mirror of containers -apptainer.registry = 'quay.io' -docker.registry = 'quay.io' -podman.registry = 'quay.io' -singularity.registry = 'quay.io' +apptainer.registry = 'quay.io' +docker.registry = 'quay.io' +podman.registry = 'quay.io' +singularity.registry = 'quay.io' +charliecloud.registry = 'quay.io' // Nextflow plugins plugins { diff --git a/nf_core/pipelines/create/__init__.py b/nf_core/pipelines/create/__init__.py index da6a693220..56e25bf1d5 100644 --- a/nf_core/pipelines/create/__init__.py +++ b/nf_core/pipelines/create/__init__.py @@ -5,6 +5,7 @@ from textual.app import App from textual.widgets import Button +from nf_core.pipelines.create import utils from nf_core.pipelines.create.basicdetails import BasicDetails from nf_core.pipelines.create.custompipeline import CustomPipeline from nf_core.pipelines.create.finaldetails import FinalDetails @@ -14,15 +15,10 @@ from nf_core.pipelines.create.loggingscreen import LoggingScreen from nf_core.pipelines.create.nfcorepipeline import NfcorePipeline from nf_core.pipelines.create.pipelinetype import ChoosePipelineType -from nf_core.pipelines.create.utils import ( - CreateConfig, - CustomLogHandler, - LoggingConsole, -) from nf_core.pipelines.create.welcome import WelcomeScreen -log_handler = CustomLogHandler( - console=LoggingConsole(classes="log_console"), +log_handler = utils.CustomLogHandler( + console=utils.LoggingConsole(classes="log_console"), rich_tracebacks=True, show_time=False, show_path=False, @@ -36,7 +32,7 @@ log_handler.setLevel("INFO") -class PipelineCreateApp(App[CreateConfig]): +class PipelineCreateApp(App[utils.CreateConfig]): """A Textual app to manage stopwatches.""" CSS_PATH = "create.tcss" @@ -60,10 +56,10 @@ class PipelineCreateApp(App[CreateConfig]): } # Initialise config as empty - TEMPLATE_CONFIG = CreateConfig() + TEMPLATE_CONFIG = utils.CreateConfig() # Initialise pipeline type - PIPELINE_TYPE = None + NFCORE_PIPELINE = True # Log handler LOG_HANDLER = log_handler @@ -78,10 +74,12 @@ def on_button_pressed(self, event: Button.Pressed) -> None: if event.button.id == "start": self.push_screen("choose_type") elif event.button.id == "type_nfcore": - self.PIPELINE_TYPE = "nfcore" + self.NFCORE_PIPELINE = True + utils.NFCORE_PIPELINE_GLOBAL = True self.push_screen("basic_details") elif event.button.id == "type_custom": - self.PIPELINE_TYPE = "custom" + self.NFCORE_PIPELINE = False + utils.NFCORE_PIPELINE_GLOBAL = False self.push_screen("basic_details") elif event.button.id == "continue": self.push_screen("final_details") diff --git a/nf_core/pipelines/create/basicdetails.py b/nf_core/pipelines/create/basicdetails.py index b88ede10d0..09484fa2ea 100644 --- a/nf_core/pipelines/create/basicdetails.py +++ b/nf_core/pipelines/create/basicdetails.py @@ -39,7 +39,7 @@ def compose(self) -> ComposeResult: "GitHub organisation", "nf-core", classes="column", - disabled=self.parent.PIPELINE_TYPE == "nfcore", + disabled=self.parent.NFCORE_PIPELINE, ) yield TextInput( "name", @@ -85,7 +85,7 @@ def on_screen_resume(self): add_hide_class(self.parent, "exist_warn") for text_input in self.query("TextInput"): if text_input.field_id == "org": - text_input.disabled = self.parent.PIPELINE_TYPE == "nfcore" + text_input.disabled = self.parent.NFCORE_PIPELINE @on(Button.Pressed) def on_button_pressed(self, event: Button.Pressed) -> None: @@ -102,9 +102,9 @@ def on_button_pressed(self, event: Button.Pressed) -> None: try: self.parent.TEMPLATE_CONFIG = CreateConfig(**config) if event.button.id == "next": - if self.parent.PIPELINE_TYPE == "nfcore": + if self.parent.NFCORE_PIPELINE: self.parent.push_screen("type_nfcore") - elif self.parent.PIPELINE_TYPE == "custom": + else: self.parent.push_screen("type_custom") except ValueError: pass diff --git a/nf_core/pipelines/create/utils.py b/nf_core/pipelines/create/utils.py index 6006452baf..f1e0bae3ce 100644 --- a/nf_core/pipelines/create/utils.py +++ b/nf_core/pipelines/create/utils.py @@ -1,9 +1,11 @@ import re +from contextlib import contextmanager +from contextvars import ContextVar from logging import LogRecord from pathlib import Path -from typing import Optional, Union +from typing import Any, Dict, Iterator, Optional, Union -from pydantic import BaseModel, ConfigDict, ValidationError, field_validator +from pydantic import BaseModel, ConfigDict, ValidationError, ValidationInfo, field_validator from rich.logging import RichHandler from textual import on from textual._context import active_app @@ -14,6 +16,22 @@ from textual.widget import Widget from textual.widgets import Button, Input, Markdown, RichLog, Static, Switch +# Use ContextVar to define a context on the model initialization +_init_context_var: ContextVar = ContextVar("_init_context_var", default={}) + + +@contextmanager +def init_context(value: Dict[str, Any]) -> Iterator[None]: + token = _init_context_var.set(value) + try: + yield + finally: + _init_context_var.reset(token) + + +# Define a global variable to store the pipeline type +NFCORE_PIPELINE_GLOBAL: bool = True + class CreateConfig(BaseModel): """Pydantic model for the nf-core create config.""" @@ -30,12 +48,25 @@ class CreateConfig(BaseModel): model_config = ConfigDict(extra="allow") + def __init__(self, /, **data: Any) -> None: + """Custom init method to allow using a context on the model initialization.""" + self.__pydantic_validator__.validate_python( + data, + self_instance=self, + context=_init_context_var.get(), + ) + @field_validator("name") @classmethod - def name_nospecialchars(cls, v: str) -> str: + def name_nospecialchars(cls, v: str, info: ValidationInfo) -> str: """Check that the pipeline name is simple.""" - if not re.match(r"^[a-z]+$", v): - raise ValueError("Must be lowercase without punctuation.") + context = info.context + if context and context["is_nfcore"]: + if not re.match(r"^[a-z]+$", v): + raise ValueError("Must be lowercase without punctuation.") + else: + if not re.match(r"^[a-zA-Z-_]+$", v): + raise ValueError("Must not contain special characters. Only '-' or '_' are allowed.") return v @field_validator("org", "description", "author", "version", "outdir") @@ -117,8 +148,9 @@ def validate(self, value: str) -> ValidationResult: If it fails, return the error messages.""" try: - CreateConfig(**{f"{self.key}": value}) - return self.success() + with init_context({"is_nfcore": NFCORE_PIPELINE_GLOBAL}): + CreateConfig(**{f"{self.key}": value}) + return self.success() except ValidationError as e: return self.failure(", ".join([err["msg"] for err in e.errors()])) diff --git a/nf_core/subworkflow-template/tests/main.nf.test.j2 b/nf_core/subworkflow-template/tests/main.nf.test.j2 index c44e19a4e4..8aaf6e0c7c 100644 --- a/nf_core/subworkflow-template/tests/main.nf.test.j2 +++ b/nf_core/subworkflow-template/tests/main.nf.test.j2 @@ -7,7 +7,7 @@ nextflow_workflow { workflow "{{ component_name_underscore|upper }}" tag "subworkflows" - tag "subworkflows_nfcore" + tag "subworkflows_{{ org_alphabet }}" tag "subworkflows/{{ component_name }}" // TODO nf-core: Add tags for all modules used within this subworkflow. Example: tag "samtools" diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 335789cb7a..601f351fda 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -4,6 +4,7 @@ import json import logging +import re from pathlib import Path import yaml @@ -192,10 +193,13 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): ) # Verify that tags are correct. main_nf_tags = subworkflow._get_main_nf_tags(subworkflow.nftest_main_nf) + not_alphabet = re.compile(r"[^a-zA-Z]") + org_alp = not_alphabet.sub("", subworkflow.org) + org_alphabet = org_alp if org_alp != "" else "nfcore" required_tags = [ "subworkflows", f"subworkflows/{subworkflow.component_name}", - "subworkflows_nfcore", + f"subworkflows_{org_alphabet}", ] included_components = [] if subworkflow.main_nf.is_file(): diff --git a/nf_core/utils.py b/nf_core/utils.py index 8c50f0a49f..1a1e8a2a99 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1069,7 +1069,7 @@ def determine_base_dir(directory="."): config_fn = get_first_available_path(base_dir, CONFIG_PATHS) if config_fn: break - return directory if base_dir == start_dir else base_dir + return directory if (base_dir == start_dir or str(base_dir) == base_dir.root) else base_dir def get_first_available_path(directory, paths):