Feature/remove functionality level

CHANGELOG.md

@@ -7,10 +7,16 @@
     config_mods: |
       .functionality.argument_groups[true].arguments[.multiple == true].multiple_sep := ":"
 
+* `functionality`: Remove the `functionality` layer from the config and move all fields to the top layer (PR #649).
+
 ## MINOR CHANGES
 
 * `package config`: Renamed `project config` to `package config` (PR #636). Now that we start using the config more, we came to the conclusion that "package" was better suited that "project".
 
+* `ns exec`: Added an extra field `{name}` to replace `{functionality-name}` (PR #649). No immediate removal of the old field is planned, but it is deprecated.
+
+* `BashWrapper`: Added meta-data field `meta_name` as a replacement for `meta_functionality_name` (PR #649). No immediate removal of the old field is planned, but it is deprecated.
+
 ## BUG FIXES
 
 * `schema`: Don't require undocumented fields to set default values and add the `links` and `reference` fields to functionality as they were not meant only to be in the project config (PR #636).

docs/reference/config_mods/index.qmd

@@ -23,7 +23,7 @@ viash COMMAND \
 Change the version of a component:
 
 ```bash
-viash build -c '.functionality.version := "0.3.0"'
+viash build -c '.version := "0.3.0"'
 ```
 
 Change the registry of a docker container:
@@ -36,14 +36,14 @@ viash build -c \
 Add an author to the list:
 
 ```bash
-viash build -c '.functionality.authors += { name: "Mr. T", role: "sponsor" }'
+viash build -c '.authors += { name: "Mr. T", role: "sponsor" }'
 ```
 
 You can use dynamic config modding to alter the config of multiple components at once:
 
 ```bash
 viash ns build \
-  -c '.functionality.version := "0.3.0"' \
+  -c '.version := "0.3.0"' \
   -c '.engines[.type == "docker"].registry := "url-to-registry"' \
-  -c '.functionality.authors += { name: "Mr. T", role: "sponsor" }'
+  -c '.authors += { name: "Mr. T", role: "sponsor" }'
 ```

docs/reference/viash_code_block/index.qmd

@@ -105,7 +105,9 @@ Meta-variables offer information on the runtime environment which you can use fr
 
 * `executable` (string): The executable being used at runtime; that is, the wrapped script. This variable is used in unit tests.
 
-* `functionality_name` (string): The name of the component, useful for logging.
+* `name` (string): The name of the component, useful for logging.
+
+* `functionality_name` (string): The name of the component, useful for logging. (Deprecated)
 
 * `memory_*` (long): The maximum amount of memory a component is allowed to allocate. The following denominations are provided: `memory_b`, `memory_kb`, `memory_mb`, `memory_gb`, `memory_tb`, `memory_pb`. By default, this value will be undefined.
 
@@ -238,10 +240,14 @@ grep -q 'expected output' output.txt
 echo Done
 ```
 
-### `functionality_name` (string)
+### `name` (string)
 
 The name of the component, useful for logging.
 
+### `functionality_name` (string)
+
+The name of the component, useful for logging. (Deprecated)
+
 ### `memory_*` (long)
 
 The maximum amount of memory a component is allowed to allocate.

src/main/resources/io/viash/helpers/bashutils/ViashDockerFuns.sh

@@ -208,7 +208,7 @@ function ViashDockerCheckCommands {
 # $1                               : image identifier with format `[registry/]image[:tag]`
 # $...                             : additional arguments to pass to docker build
 # $VIASH_META_TEMP_DIR             : temporary directory to store dockerfile & optional resources in
-# $VIASH_META_FUNCTIONALITY_NAME   : name of the component
+# $VIASH_META_NAME                 : name of the component
 # $VIASH_META_RESOURCES_DIR        : directory containing the resources
 # $VIASH_VERBOSITY                 : verbosity level
 # exit code $?                     : whether or not the image was built successfully
@@ -217,7 +217,7 @@ function ViashDockerBuild {
   shift 1
 
   # create temporary directory to store dockerfile & optional resources in
-  local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_FUNCTIONALITY_NAME-XXXXXX")
+  local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX")
   local dockerfile="$tmpdir/Dockerfile"
   function clean_up {
     rm -rf "$tmpdir"

src/main/resources/io/viash/runners/nextflow/VDSL3Helper.nf

@@ -34,7 +34,7 @@ def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) {
         }
 
         // process input files separately
-        def inputPaths = meta.config.functionality.allArguments
+        def inputPaths = meta.config.allArguments
           .findAll { it.type == "file" && it.direction == "input" }
           .collect { par ->
             def val = data_.containsKey(par.plainName) ? data_[par.plainName] : []
@@ -62,7 +62,7 @@ def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) {
           } 
 
         // remove input files
-        def argsExclInputFiles = meta.config.functionality.allArguments
+        def argsExclInputFiles = meta.config.allArguments
           .findAll { (it.type != "file" || it.direction != "input") && data_.containsKey(it.plainName) }
           .collectEntries { par ->
             def parName = par.plainName
@@ -80,7 +80,7 @@ def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) {
       }
       | processObj
       | map { output ->
-        def outputFiles = meta.config.functionality.allArguments
+        def outputFiles = meta.config.allArguments
           .findAll { it.type == "file" && it.direction == "output" }
           .indexed()
           .collectEntries{ index, par ->
@@ -174,12 +174,12 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
     }
   }.join()
 
-  def inputPaths = meta.config.functionality.allArguments
+  def inputPaths = meta.config.allArguments
     .findAll { it.type == "file" && it.direction == "input" }
     .collect { ', path(viash_par_' + it.plainName + ', stageAs: "_viash_par/' + it.plainName + '_?/*")' }
     .join()
 
-  def outputPaths = meta.config.functionality.allArguments
+  def outputPaths = meta.config.allArguments
     .findAll { it.type == "file" && it.direction == "output" }
     .collect { par ->
       // insert dummy into every output (see nextflow-io/nextflow#2678)
@@ -199,15 +199,15 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
   }
 
   // create dirs for output files (based on BashWrapper.createParentFiles)
-  def createParentStr = meta.config.functionality.allArguments
+  def createParentStr = meta.config.allArguments
     .findAll { it.type == "file" && it.direction == "output" && it.create_parent }
     .collect { par -> 
       "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"] : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }"
     }
     .join("\n")
 
   // construct inputFileExports
-  def inputFileExports = meta.config.functionality.allArguments
+  def inputFileExports = meta.config.allArguments
     .findAll { it.type == "file" && it.direction.toLowerCase() == "input" }
     .collect { par ->
       viash_par_contents = "(viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName})"
@@ -229,7 +229,7 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
   ).toAbsolutePath()
 
   // construct stub
-  def stub = meta.config.functionality.allArguments
+  def stub = meta.config.allArguments
     .findAll { it.type == "file" && it.direction == "output" }
     .collect { par -> 
       "\${ args.containsKey(\"${par.plainName}\") ? \"touch2 \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"].replace(\"_*\", \"_0\") : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }"
@@ -269,8 +269,8 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
   |# export VIASH_META_RESOURCES_DIR="\${resourcesDir.toRealPath().toAbsolutePath()}"
   |export VIASH_META_RESOURCES_DIR="\${resourcesDir}"
   |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}"
-  |export VIASH_META_FUNCTIONALITY_NAME="${meta.config.functionality.name}"
-  |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_FUNCTIONALITY_NAME"
+  |export VIASH_META_NAME="${meta.config.name}"
+  |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_NAME"
   |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml"
   |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" }
   |\${task.memory?.bytes != null ? "export VIASH_META_MEMORY_B=\$task.memory.bytes" : "" }

src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf

@@ -1,14 +1,14 @@
 Map _processInputValues(Map inputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
-    config.functionality.allArguments.each { arg ->
+    config.allArguments.each { arg ->
       if (arg.required) {
         assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : 
           "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing"
       }
     }
 
     inputs = inputs.collectEntries { name, value ->
-      def par = config.functionality.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") }
+      def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") }
       assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument"
 
       value = _checkArgumentType("input", par, value, "in module '$key' id '$id'")

src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf

@@ -1,14 +1,14 @@
 Map _processOutputValues(Map outputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
-    config.functionality.allArguments.each { arg ->
+    config.allArguments.each { arg ->
       if (arg.direction == "output" && arg.required) {
         assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
           "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
       }
     }
 
     outputs = outputs.collectEntries { name, value ->
-      def par = config.functionality.allArguments.find { it.plainName == name && it.direction == "output" }
+      def par = config.allArguments.find { it.plainName == name && it.direction == "output" }
       assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument"
 
       value = _checkArgumentType("output", par, value, "in module '$key' id '$id'")

src/main/resources/io/viash/runners/nextflow/channel/_parseParamList.nf

@@ -65,7 +65,7 @@ def _parseParamList(param_list, Map config) {
   }
 
   // id is argument
-  def idIsArgument = config.functionality.allArguments.any{it.plainName == "id"}
+  def idIsArgument = config.allArguments.any{it.plainName == "id"}
 
   // Reformat from List<Map> to List<Tuple2<String, Map>> by adding the ID as first element of a Tuple2
   paramSets = paramSets.collect({ data ->
@@ -87,7 +87,7 @@ def _parseParamList(param_list, Map config) {
   if (paramListPath) {
     paramSets = paramSets.collect({ id, data ->
       def new_data = data.collectEntries{ parName, parValue ->
-        def par = config.functionality.allArguments.find{it.plainName == parName}
+        def par = config.allArguments.find{it.plainName == parName}
         if (par && par.type == "file" && par.direction == "input") {
           if (parValue instanceof Collection) {
             parValue = parValue.collectMany{path -> 

src/main/resources/io/viash/runners/nextflow/channel/_splitParams.nf

@@ -10,7 +10,7 @@
  */
 Map<String, Object> _splitParams(Map<String, Object> parValues, Map config){
   def parsedParamValues = parValues.collectEntries { parName, parValue ->
-    def parameterSettings = config.functionality.allArguments.find({it.plainName == parName})
+    def parameterSettings = config.allArguments.find({it.plainName == parName})
 
     if (!parameterSettings) {
       // if argument is not found, do not alter 

src/main/resources/io/viash/runners/nextflow/channel/channelFromParams.nf

@@ -31,11 +31,11 @@
 
 private List<Tuple2<String, Map<String, Object>>> _paramsToParamSets(Map params, Map config){
   // todo: fetch key from run args
-  def key_ = config.functionality.name
+  def key_ = config.name
 
   /* parse regular parameters (not in param_list)  */
   /*************************************************/
-  def globalParams = config.functionality.allArguments
+  def globalParams = config.allArguments
     .findAll { params.containsKey(it.plainName) }
     .collectEntries { [ it.plainName, params[it.plainName] ] }
   def globalID = params.get("id", null)
@@ -76,7 +76,7 @@ private List<Tuple2<String, Map<String, Object>>> _paramsToParamSets(Map params,
     //   }
     // }
     parValues = parValues.collectEntries { name, value ->
-      def par = config.functionality.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") }
+      def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") }
       assert par != null : "Error in module '${key_}' id '${id}': '${name}' is not a valid input argument"
 
       if (par == null) {

src/main/resources/io/viash/runners/nextflow/channel/preprocessInputs.nf

@@ -38,10 +38,10 @@ def preprocessInputs(Map args) {
   assert config instanceof Map : 
     "Error in preprocessInputs: config must be a map. " +
     "Expected class: Map. Found: config.getClass() is ${config.getClass()}"
-  def key_ = args.key ?: config.functionality.name
+  def key_ = args.key ?: config.name
 
   // Get different parameter types (used throughout this function)
-  def defaultArgs = config.functionality.allArguments
+  def defaultArgs = config.allArguments
     .findAll { it.containsKey("default") }
     .collectEntries { [ it.plainName, it.default ] }
 
@@ -51,7 +51,7 @@ def preprocessInputs(Map args) {
     def passthrough = tup.drop(2)
 
     def new_data = (defaultArgs + data).collectEntries { name, value ->
-      def par = config.functionality.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") }
+      def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") }
 
       if (par != null) {
         value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'")

src/main/resources/io/viash/runners/nextflow/config/addGlobalParams.nf

@@ -1,39 +1,37 @@
 def addGlobalArguments(config) {
   def localConfig = [
-    "functionality" : [
-      "argument_groups": [
-        [
-          "name": "Nextflow input-output arguments",
-          "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
-          "arguments" : [
-            [
-              'name': '--publish_dir',
-              'required': true,
-              'type': 'string',
-              'description': 'Path to an output directory.',
-              'example': 'output/',
-              'multiple': false
-            ],
-            [
-              'name': '--param_list',
-              'required': false,
-              'type': 'string',
-              'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob.
-              |
-              |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`.
-              |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`.
-              |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`.
-              |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`.
-              |
-              |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(),
-              'example': 'my_params.yaml',
-              'multiple': false,
-              'hidden': true
-            ]
-            // TODO: allow multiple: true in param_list?
-            // TODO: allow to specify a --param_list_regex to filter the param_list?
-            // TODO: allow to specify a --param_list_from_state to remap entries in the param_list?
+    "argument_groups": [
+      [
+        "name": "Nextflow input-output arguments",
+        "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
+        "arguments" : [
+          [
+            'name': '--publish_dir',
+            'required': true,
+            'type': 'string',
+            'description': 'Path to an output directory.',
+            'example': 'output/',
+            'multiple': false
+          ],
+          [
+            'name': '--param_list',
+            'required': false,
+            'type': 'string',
+            'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob.
+            |
+            |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`.
+            |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`.
+            |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`.
+            |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`.
+            |
+            |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(),
+            'example': 'my_params.yaml',
+            'multiple': false,
+            'hidden': true
           ]
+          // TODO: allow multiple: true in param_list?
+          // TODO: allow to specify a --param_list_regex to filter the param_list?
+          // TODO: allow to specify a --param_list_from_state to remap entries in the param_list?
         ]
       ]
     ]

src/main/resources/io/viash/runners/nextflow/config/generateHelp.nf

@@ -60,7 +60,7 @@ def _generateArgumentHelp(param) {
 
 // Based on Helper.generateHelp() in Helper.scala
 def _generateHelp(config) {
-  def fun = config.functionality
+  def fun = config
 
   // PART 1: NAME AND VERSION
   def nameStr = fun.name +