Merge remote-tracking branch 'origin/develop' into anonymize_config_o…

…utput

CHANGELOG.md

@@ -18,6 +18,8 @@ TODO add summary
 
 * `config view` and `ns list`: Do not output internal functionality fields (#564). Additionally, add a validation that no internal fields are present when reading a Viash config file.
 
+* `project config`: Add fields in the project config to specify default values for component config fields (PR #612). This allows for a more DRY approach to defining the same values for multiple components.
+
 ## MINOR CHANGES
 
 * `testbenches`: Add testbenches for local dependencies (PR #565).

src/main/scala/io/viash/Main.scala

@@ -25,7 +25,7 @@ import sys.process.{Process, ProcessLogger}
 import config.Config
 import helpers.{IO, Exec, SysEnv, DependencyResolver, Logger, Logging}
 import helpers.status._
-import project.ViashProject
+import project.ProjectConfig
 import cli.{CLIConf, ViashCommand, DocumentedSubcommand, ViashNs, ViashNsBuild, ViashLogger}
 import exceptions._
 import scala.util.Try
@@ -168,7 +168,7 @@ object Main extends Logging {
     */
   def mainCLI(args: Array[String], workingDir: Option[Path] = None): Int = {
     // try to find project settings
-    val proj0 = workingDir.map(ViashProject.findViashProject(_)).getOrElse(ViashProject())
+    val proj0 = workingDir.map(ProjectConfig.findViashProject(_)).getOrElse(ProjectConfig())
 
     // strip arguments meant for viash run
     val (viashArgs, runArgs) = {
@@ -429,16 +429,15 @@ object Main extends Logging {
 
   def readConfig(
     subcommand: ViashCommand,
-    project: ViashProject,
+    project: ProjectConfig,
     addOptMainScript: Boolean = true,
     applyRunnerAndEngine: Boolean = true
   ): AppliedConfig = {
 
     val config = Config.read(
       configPath = subcommand.config(),
-      projectDir = project.rootDir.map(_.toUri()),
       addOptMainScript = addOptMainScript,
-      configMods = project.config_mods
+      viashProject = Some(project)
     )
     if (applyRunnerAndEngine) {
       val runnerStr = subcommand.runner.toOption
@@ -460,7 +459,7 @@ object Main extends Logging {
 
   def readConfigs(
     subcommand: ViashNs,
-    project: ViashProject,
+    project: ProjectConfig,
     addOptMainScript: Boolean = true,
     applyRunner: Boolean = true,
     applyEngine: Boolean = true
@@ -475,12 +474,11 @@ object Main extends Logging {
 
     val configs0 = Config.readConfigs(
       source = source,
-      projectDir = project.rootDir.map(_.toUri()),
       query = query,
       queryNamespace = queryNamespace,
       queryName = queryName,
-      configMods = configMods,
-      addOptMainScript = addOptMainScript
+      addOptMainScript = addOptMainScript,
+      viashProject = Some(project)
     )
 
     // TODO: apply engine and runner should probably be split into two Y_Y
@@ -578,8 +576,8 @@ object Main extends Logging {
       // else look for project file in working dir
       // and try to read as json
       workingDir
-        .flatMap(ViashProject.findProjectFile)
-        .map(ViashProject.readJson)
+        .flatMap(ProjectConfig.findProjectFile)
+        .map(ProjectConfig.readJson)
         .flatMap(js => {
           js.asObject.flatMap(_.apply("viash_version")).flatMap(_.asString)
         })

src/main/scala/io/viash/config/AppliedConfig.scala

@@ -41,7 +41,7 @@ final case class AppliedConfig(
 }
 
 object AppliedConfig {
-  implicit def fromConfig(config: Config) = {
+  implicit def fromConfig(config: Config): AppliedConfig = {
     AppliedConfig(config, None, Nil, None)
   }
 }

src/main/scala/io/viash/config/Config.scala

@@ -38,6 +38,8 @@ import java.nio.file.FileSystemNotFoundException
 import io.viash.runners.{Runner, ExecutableRunner, NextflowRunner}
 import io.viash.engines.{Engine, NativeEngine, DockerEngine}
 import io.viash.helpers.ReplayableMultiOutputStream
+import io.viash.project.ProjectConfig
+import io.viash.lenses.ConfigLenses._
 
 @description(
   """A Viash configuration is a YAML file which contains metadata to describe the behaviour and build target(s) of a component.  
@@ -89,8 +91,10 @@ case class Config(
   @undocumented
   info: Option[Info] = None,
 
-  @internalFunctionality
-  projectDir: Option[String] = None,
+  @description("The project config content used during build.")
+  @since("Viash 0.9.0")
+  @undocumented
+  project_config: Option[ProjectConfig] = None,
 ) {
 
   @description(
@@ -239,25 +243,25 @@ object Config extends Logging {
   // reads and modifies the config based on the current setup
   def read(
     configPath: String,
-    projectDir: Option[URI] = None,
     addOptMainScript: Boolean = true,
-    configMods: List[String] = Nil
+    viashProject: Option[ProjectConfig] = None,
   ): Config = {
     val uri = IO.uri(configPath)
     readFromUri(
       uri = uri,
-      projectDir = projectDir,
       addOptMainScript = addOptMainScript,
-      configMods = configMods
+      viashProject = viashProject
     )
   }
 
   def readFromUri(
     uri: URI,
-    projectDir: Option[URI] = None,
     addOptMainScript: Boolean = true,
-    configMods: List[String] = Nil
+    viashProject: Option[ProjectConfig] = None,
   ): Config = {
+    val projectDir = viashProject.flatMap(_.rootDir).map(_.toUri())
+    val configMods: List[String] = viashProject.map(_.config_mods.toList).getOrElse(Nil)
+
     // read cli config mods
     val confMods = ConfigMods.parseConfigMods(configMods)
 
@@ -280,9 +284,40 @@ object Config extends Logging {
     // apply preparse config mods if need be
     val json2 = confMods(json1, preparse = true)
 
-    /* CONFIG 0: converted from json */
+    /* CONFIG Base: converted from json */
     // convert Json into Config
-    val conf0 = Convert.jsonToClass[Config](json2, uri.toString())
+    val confBase = Convert.jsonToClass[Config](json2, uri.toString())
+
+    /* CONFIG 0: apply values from project config */
+    // apply values from project config if need be
+    val conf0 = {
+      val vpVersion = viashProject.flatMap(_.version)
+      val vpRepositories = viashProject.map(_.repositories).getOrElse(Nil)
+      val vpLicense = viashProject.flatMap(_.license)
+      val vpOrganization = viashProject.flatMap(_.organization)
+      val vpRepository = viashProject.flatMap(_.links.repository)
+      val vpDockerRegistry = viashProject.flatMap(_.links.docker_registry)
+
+      val mappedEngines = confBase.engines.map {
+        _ match {
+          case e: DockerEngine =>
+            e.copy(
+              target_image_source = e.target_image_source.orElse(vpRepository),
+              target_registry = e.target_registry.orElse(vpDockerRegistry)
+            )
+          case e => e
+        }
+      }
+
+      val lenses =
+        composedVersionLens.modify(_ orElse vpVersion) andThen
+        composedLicenseLens.modify(_ orElse vpLicense) andThen
+        composedOrganizationLens.modify(_ orElse vpOrganization) andThen
+        composedRepositoriesLens.modify(vpRepositories ::: _) andThen
+        enginesLens.set(mappedEngines)
+
+      lenses(confBase)
+    }
 
     /* CONFIG 1: apply post-parse config mods */
     // apply config mods only if need be
@@ -293,24 +328,19 @@ object Config extends Logging {
         // apply config mods
         val modifiedJs = confMods(js, preparse = false)
         // turn json back into a config
-        modifiedJs.as[Config].fold(throw _, identity)
+        Convert.jsonToClass[Config](modifiedJs, uri.toString())
       } else {
         conf0
       }
 
-    /* CONFIG 1: store parent path in resource to be able to access them in the future */
+    /* CONFIG 2: store parent path in resource to be able to access them in the future */
+    // copy resources with updated paths into config and return
     val parentURI = uri.resolve("")
     val resources = conf1.functionality.resources.map(_.copyWithAbsolutePath(parentURI, projectDir))
     val tests = conf1.functionality.test_resources.map(_.copyWithAbsolutePath(parentURI, projectDir))
 
-    // copy resources with updated paths into config and return
-    val conf2 = conf1.copy(
-      functionality = conf0.functionality.copy(
-        resources = resources,
-        test_resources = tests
-      ),
-      projectDir = projectDir.map(_.toString.replaceAll("^file:/+", "/"))
-    )
+    val conf2a = composedResourcesLens.set(resources)(conf1)
+    val conf2 = composedTestResourcesLens.set(tests)(conf2a)
 
     /* CONFIG 3: add info */
     // gather git info
@@ -337,35 +367,35 @@ object Config extends Logging {
     }
 
     // print warnings if need be
-    if (conf2.functionality.status == Status.Deprecated)
-      warn(s"Warning: The status of the component '${conf2.functionality.name}' is set to deprecated.")
+    if (conf3.functionality.status == Status.Deprecated)
+      warn(s"Warning: The status of the component '${conf3.functionality.name}' is set to deprecated.")
 
-    if (conf2.functionality.resources.isEmpty && optScript.isEmpty)
+    if (conf3.functionality.resources.isEmpty && optScript.isEmpty)
       warn(s"Warning: no resources specified!")
 
+    /* CONFIG 4: add Viash Project Config */
+    val conf4 = conf3.copy(
+      project_config = viashProject
+    )
+
     if (!addOptMainScript) {
-      return conf3
+      return conf4
     }
 
-    /* CONFIG 4: add main script if config is stored inside script */
+    /* CONFIG 5: add main script if config is stored inside script */
     // add info and additional resources
-    val conf4 = conf3.copy(
-      functionality = conf3.functionality.copy(
-        resources = optScript.toList ::: conf3.functionality.resources
-      )
-    )
+    val conf5 = composedResourcesLens.modify(optScript.toList ::: _)(conf4)
 
-    conf4
+    conf5
   }
 
   def readConfigs(
     source: String,
-    projectDir: Option[URI] = None,
     query: Option[String] = None,
     queryNamespace: Option[String] = None,
     queryName: Option[String] = None,
-    configMods: List[String] = Nil,
-    addOptMainScript: Boolean = true
+    addOptMainScript: Boolean = true,
+    viashProject: Option[ProjectConfig] = None,
   ): List[AppliedConfig] = {
 
     val sourceDir = Paths.get(source)
@@ -386,11 +416,10 @@ object Config extends Logging {
           // When replaying, output directly to the console. Logger functions were already called, so we don't want to call them again.
           Console.withErr(rmos.getOutputStream(Console.err.print)) {
             Console.withOut(rmos.getOutputStream(Console.out.print)) {
-              Config.read(
+              read(
                 file.toString,
-                projectDir = projectDir,
                 addOptMainScript = addOptMainScript,
-                configMods = configMods
+                viashProject = viashProject
               )
             }
           }

src/main/scala/io/viash/config/ConfigMeta.scala

@@ -42,18 +42,18 @@ object ConfigMeta {
 
   def configToCleanJson(config: Config): Json = {
     // relativize paths in the info field
-    val anonymizedConfig = config.projectDir match {
-      case Some(dir) =>
-        val path = Paths.get(dir)
-        config.copy(
-          info = config.info.map(info => info.copy(
-            config = IO.anonymizePath(dir, info.config),
-            output = info.output.map(o => IO.anonymizePath(dir, o)),
-            executable = info.executable.map(e => IO.anonymizePath(dir, e))
-          ))
-        )
-      case None => config
-    }
+    val rootDir = config.project_config.flatMap(_.rootDir)
+    val anonymizedConfig = config.copy(
+      info = config.info.map(info => info.copy(
+        config = IO.anonymizePath(rootDir, info.config),
+        output = info.output.map(IO.anonymizePath(rootDir, _)),
+        executable = info.executable.map(IO.anonymizePath(rootDir, _))
+      )),
+      project_config = config.project_config.map(pc => pc.copy(
+        source = pc.source.map(IO.anonymizePath(rootDir, _)),
+        target = pc.target.map(IO.anonymizePath(rootDir, _))
+      ))
+    )
 
     val encodedConfig: Json = encodeConfig(anonymizedConfig)
     // drop empty & null values recursively except all "info" fields

src/main/scala/io/viash/config/Info.scala

@@ -18,6 +18,7 @@
 package io.viash.config
 
 import io.viash.schemas.description
+import io.viash.project.ProjectConfig
 
 // TODO: rename to ConfigInfo?
 
@@ -40,5 +41,5 @@ case class Info(
   @description("Git remote name.")
   git_remote: Option[String] = None,
   @description("Git tag.")
-  git_tag: Option[String] = None
+  git_tag: Option[String] = None,
 )

src/main/scala/io/viash/functionality/Functionality.scala

@@ -249,6 +249,24 @@ case class Functionality(
       "yaml")
   @default("Empty")
   repositories: List[RepositoryWithName] = Nil,
+
+  @description("The keywords of the components.")
+  @example("keywords: [ bioinformatics, genomics ]", "yaml")
+  @default("Empty")
+  @since("Viash 0.9.0")
+  keywords: List[String] = Nil,
+
+  @description("The license of the project.")
+  @example("license: MIT", "yaml")
+  @default("Empty")
+  @since("Viash 0.9.0")
+  license: Option[String] = None,
+
+  @description("The organization of the project.")
+  @example("organization: viash-io", "yaml")
+  @default("Empty")
+  @since("Viash 0.9.0")
+  organization: Option[String] = None,
   // The variables below are for internal use and shouldn't be publicly documented
 
   // setting this to true will change the working directory

src/main/scala/io/viash/functionality/dependencies/package.scala

@@ -51,10 +51,10 @@ package object dependencies {
       objJson deepMerge typeJson
   }
 
-  implicit val encodeGitRepositoryWithName: Encoder.AsObject[GitRepositoryWithName] = deriveConfiguredEncoder
-  implicit val encodeGithubRepositoryWithName: Encoder.AsObject[GithubRepositoryWithName] = deriveConfiguredEncoder
-  implicit val encodeViashhubRepositoryWithName: Encoder.AsObject[ViashhubRepositoryWithName] = deriveConfiguredEncoder
-  implicit val encodeLocalRepositoryWithName: Encoder.AsObject[LocalRepositoryWithName] = deriveConfiguredEncoder
+  implicit val encodeGitRepositoryWithName: Encoder.AsObject[GitRepositoryWithName] = deriveConfiguredEncoderStrict
+  implicit val encodeGithubRepositoryWithName: Encoder.AsObject[GithubRepositoryWithName] = deriveConfiguredEncoderStrict
+  implicit val encodeViashhubRepositoryWithName: Encoder.AsObject[ViashhubRepositoryWithName] = deriveConfiguredEncoderStrict
+  implicit val encodeLocalRepositoryWithName: Encoder.AsObject[LocalRepositoryWithName] = deriveConfiguredEncoderStrict
   implicit def encodeRepositoryWithName[A <: RepositoryWithName]: Encoder[A] = Encoder.instance {
     par =>
       val typeJson = Json.obj("type" -> Json.fromString(par.`type`))

src/main/scala/io/viash/helpers/IO.scala

@@ -339,15 +339,17 @@ object IO extends Logging {
     * @param path The path to relativize
     * @return A relativized path or the original path if it was not relative to the base path
     */
-  def anonymizePath(basePath: String, path: String): String = {
-    val rootPath = Paths.get(basePath)
+  def anonymizePath(basePath: Option[Path], path: String): String = {    
     val pathPath = Paths.get(path)
-    val relative = rootPath.relativize(pathPath).toString()
-
-    if (relative.startsWith("..")) {
-      Paths.get("[anonymized]", pathPath.toFile().getName()).toString()
-    } else {
-      relative
+    val relative = basePath.map(_.relativize(pathPath).toString())
+
+    println(s"anonymizePath: $basePath, $path, $relative")
+
+    relative match {
+      case Some(rel) if rel.startsWith("..") => Paths.get("[anonymized]", pathPath.toFile().getName()).toString()
+      case Some(rel) => rel
+      case None if pathPath.isAbsolute => Paths.get("[anonymized]", pathPath.toFile().getName()).toString()
+      case None => path
     }
   }
 }