📦 Add new command nf-core rocrate to create a Research Object (RO) crate for a pipeline (continuation from #2680) [skip changelog]

.github/actions/create-lint-wf/action.yml

@@ -65,6 +65,12 @@ runs:
       run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
       working-directory: create-lint-wf
 
+    # Add empty ro-crate file
+    - name: add empty ro-crate file
+      shell: bash
+      run: touch nf-core-testpipeline/ro-crate-metadata.json
+      working-directory: create-lint-wf
+
     # Run nf-core pipelines linting
     - name: nf-core pipelines lint
       shell: bash

.github/workflows/create-test-lint-wf-template.yml

@@ -147,6 +147,11 @@ jobs:
         run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
         working-directory: create-test-lint-wf
 
+      # Add empty ro-crate file
+      - name: add empty ro-crate file
+        run: touch my-prefix-testpipeline/ro-crate-metadata.json
+        working-directory: create-test-lint-wf
+
       # Run nf-core linting
       - name: nf-core pipelines lint
         run: nf-core --log-file log.txt --hide-progress pipelines lint --dir my-prefix-testpipeline --fail-warned

.github/workflows/pytest.yml

@@ -132,7 +132,7 @@ jobs:
 
       - name: Test with pytest
         run: |
-          python3 -m pytest tests/${{matrix.test}} --color=yes --cov --durations=0 && exit_code=0|| exit_code=$?
+          python3 -m pytest tests/${{matrix.test}} --color=yes --cov --cov-config=.coveragerc --durations=0 && exit_code=0|| exit_code=$?
           # don't fail if no tests were collected, e.g. for test_licence.py
           if [ "${exit_code}" -eq 5 ]; then
             echo "No tests were collected"

nf_core/__main__.py

@@ -36,6 +36,7 @@
     pipelines_launch,
     pipelines_lint,
     pipelines_list,
+    pipelines_rocrate,
     pipelines_schema_build,
     pipelines_schema_docs,
     pipelines_schema_lint,
@@ -86,7 +87,7 @@
         },
         {
             "name": "For developers",
-            "commands": ["create", "lint", "bump-version", "sync", "schema", "create-logo"],
+            "commands": ["create", "lint", "bump-version", "sync", "schema", "rocrate", "create-logo"],
         },
     ],
     "nf-core modules": [
@@ -570,6 +571,44 @@ def command_pipelines_list(ctx, keywords, sort, json, show_archived):
     pipelines_list(ctx, keywords, sort, json, show_archived)
 
 
+# nf-core pipelines rocrate
+@pipelines.command("rocrate")
+@click.argument(
+    "pipeline_dir",
+    type=click.Path(exists=True),
+    default=Path.cwd(),
+    required=True,
+    metavar="<pipeline directory>",
+)
+@click.option(
+    "-j",
+    "--json_path",
+    default=Path.cwd(),
+    type=str,
+    help="Path to save RO Crate metadata json file to",
+)
+@click.option("-z", "--zip_path", type=str, help="Path to save RO Crate zip file to")
+@click.option(
+    "-pv",
+    "--pipeline_version",
+    type=str,
+    help="Version of pipeline to use for RO Crate",
+    default="",
+)
+@click.pass_context
+def rocrate(
+    ctx,
+    pipeline_dir: str,
+    json_path: str,
+    zip_path: str,
+    pipeline_version: str,
+):
+    """
+    Make an Research Object Crate
+    """
+    pipelines_rocrate(ctx, pipeline_dir, json_path, zip_path, pipeline_version)
+
+
 # nf-core pipelines sync
 @pipelines.command("sync")
 @click.pass_context
@@ -1758,7 +1797,7 @@ def command_schema_validate(pipeline, params):
 @click.option(
     "--url",
     type=str,
-    default="https://nf-co.re/pipeline_schema_builder",
+    default="https://oldsite.nf-co.re/pipeline_schema_builder",
     help="Customise the builder URL (for development work)",
 )
 def command_schema_build(directory, no_prompts, web_only, url):

nf_core/commands_pipelines.py

@@ -2,6 +2,7 @@
 import os
 import sys
 from pathlib import Path
+from typing import Optional, Union
 
 import rich
 
@@ -277,6 +278,33 @@ def pipelines_list(ctx, keywords, sort, json, show_archived):
     stdout.print(list_workflows(keywords, sort, json, show_archived))
 
 
+# nf-core pipelines rocrate
+def pipelines_rocrate(
+    ctx,
+    pipeline_dir: Union[str, Path],
+    json_path: Optional[Union[str, Path]],
+    zip_path: Optional[Union[str, Path]],
+    pipeline_version: str,
+) -> None:
+    from nf_core.pipelines.rocrate import ROCrate
+
+    if json_path is None and zip_path is None:
+        log.error("Either `--json_path` or `--zip_path` must be specified.")
+        sys.exit(1)
+    else:
+        pipeline_dir = Path(pipeline_dir)
+        if json_path is not None:
+            json_path = Path(json_path)
+        if zip_path is not None:
+            zip_path = Path(zip_path)
+        try:
+            rocrate_obj = ROCrate(pipeline_dir, pipeline_version)
+            rocrate_obj.create_rocrate(json_path=json_path, zip_path=zip_path)
+        except (UserWarning, LookupError, FileNotFoundError) as e:
+            log.error(e)
+            sys.exit(1)
+
+
 # nf-core pipelines sync
 def pipelines_sync(ctx, directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr):
     """

nf_core/components/info.py

@@ -211,9 +211,9 @@ def get_local_yaml(self) -> Optional[Dict]:
                         return yaml.safe_load(fh)
             log.debug(f"{self.component_type[:-1].title()} '{self.component}' meta.yml not found locally")
 
-        return None
+        return {}
 
-    def get_remote_yaml(self) -> Optional[dict]:
+    def get_remote_yaml(self) -> Optional[Dict]:
         """Attempt to get the meta.yml file from a remote repo.
 
         Returns:

nf_core/modules/modules_json.py

@@ -1129,8 +1129,10 @@ def dump(self, run_prettier: bool = False) -> None:
         """
         Sort the modules.json, and write it to file
         """
+        # Sort the modules.json
+        if self.modules_json is None:
+            self.load()
         if self.modules_json is not None:
-            # Sort the modules.json
             self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"])
             if run_prettier:
                 dump_json_with_prettier(self.modules_json_path, self.modules_json)

nf_core/pipelines/create/create.py

@@ -21,6 +21,7 @@
 from nf_core.pipelines.create.utils import CreateConfig, features_yml_path, load_features_yaml
 from nf_core.pipelines.create_logo import create_logo
 from nf_core.pipelines.lint_utils import run_prettier_on_file
+from nf_core.pipelines.rocrate import ROCrate
 from nf_core.utils import LintConfigType, NFCoreTemplateConfig
 
 log = logging.getLogger(__name__)
@@ -356,6 +357,11 @@ def render_template(self) -> None:
             # Make a logo and save it, if it is a nf-core pipeline
             self.make_pipeline_logo()
 
+        if self.config.skip_features is None or "ro-crate" not in self.config.skip_features:
+            # Create the RO-Crate metadata file
+            rocrate_obj = ROCrate(self.outdir)
+            rocrate_obj.create_rocrate(json_path=self.outdir / "ro-crate-metadata.json")
+
         # Update the .nf-core.yml with linting configurations
         self.fix_linting()
 

nf_core/pipelines/create/template_features.yml

@@ -432,6 +432,16 @@ seqera_platform:
     You can extend this file adding any other desired configuration.
   nfcore_pipelines: False
   custom_pipelines: True
+rocrate:
+  skippable_paths:
+    - "ro-crate-metadata.json"
+  short_description: "Add RO-Crate metadata"
+  description: "Add a RO-Crate metadata file to describe the pipeline"
+  help_text: |
+    RO-Crate is a metadata specification to describe research data and software.
+    This will add a `ro-crate-metadata.json` file to describe the pipeline.
+  nfcore_pipelines: False
+  custom_pipelines: True
 vscode:
   skippable_paths:
     - ".vscode"

nf_core/pipelines/lint/files_exist.py

@@ -66,6 +66,7 @@ def files_exist(self) -> Dict[str, List[str]]:
         conf/igenomes.config
         .github/workflows/awstest.yml
         .github/workflows/awsfulltest.yml
+        ro-crate-metadata.json
 
     Files that *must not* be present, due to being renamed or removed in the template:
 
@@ -171,6 +172,7 @@ def files_exist(self) -> Dict[str, List[str]]:
         [Path(".github", "workflows", "awstest.yml")],
         [Path(".github", "workflows", "awsfulltest.yml")],
         [Path("modules.json")],
+        [Path("ro-crate-metadata.json")],
     ]
 
     # List of strings. Fails / warns if any of the strings exist.
@@ -198,6 +200,12 @@ def files_exist(self) -> Dict[str, List[str]]:
     ]
     files_warn_ifexists = [Path(".travis.yml")]
 
+    files_hint = [
+        [
+            ["ro-crate-metadata.json"],
+            ". Run `nf-core rocrate` to generate this file. Read more about RO-Crates in the [nf-core/tools docs](https://nf-co.re/tools#create-a-ro-crate-metadata-file).",
+        ],
+    ]
     # Remove files that should be ignored according to the linting config
     ignore_files = self.lint_config.get("files_exist", []) if self.lint_config is not None else []
 
@@ -225,7 +233,11 @@ def pf(file_path: Union[str, Path]) -> Path:
         if any([pf(f).is_file() for f in files]):
             passed.append(f"File found: {self._wrap_quotes(files)}")
         else:
-            warned.append(f"File not found: {self._wrap_quotes(files)}")
+            hint = ""
+            for file_hint in files_hint:
+                if file_hint[0] == files:
+                    hint = str(file_hint[1])
+            warned.append(f"File not found: {self._wrap_quotes(files)}{hint}")
 
     # Files that cause an error if they exist
     for file in files_fail_ifexists: