Merge pull request #625 from viash-io/anonymize_config_output

Anonymize config output

CHANGELOG.md

@@ -34,6 +34,8 @@ TODO add summary
 
 * `__merge__`: Handle invalid yaml during merging (PR #570). There was not enough error handling during this operation. Switched to the more advanced `Convert.textToJson` helper method.
 
+* `config`: Anonymize paths in the config when outputting the config (PR #625).
+
 # Viash 0.8.4 (2024-01-15): Bug fix
 
 Fix building components with dependencies that have symlinks in their paths.

src/main/scala/io/viash/config/ConfigMeta.scala

@@ -26,6 +26,7 @@ import io.viash.helpers.circe._
 import io.circe.Json
 import io.circe.JsonObject
 import io.viash.functionality.resources.NextflowScript
+import io.viash.helpers.IO
 
 object ConfigMeta {
   // create a yaml printer for writing the viash.yaml file
@@ -40,7 +41,21 @@ object ConfigMeta {
   val metaFilename: String = ".config.vsh.yaml"
 
   def configToCleanJson(config: Config): Json = {
-    val encodedConfig: Json = encodeConfig(config)
+    // relativize paths in the info field
+    val rootDir = config.project_config.flatMap(_.rootDir)
+    val anonymizedConfig = config.copy(
+      info = config.info.map(info => info.copy(
+        config = IO.anonymizePath(rootDir, info.config),
+        output = info.output.map(IO.anonymizePath(rootDir, _)),
+        executable = info.executable.map(IO.anonymizePath(rootDir, _))
+      )),
+      project_config = config.project_config.map(pc => pc.copy(
+        source = pc.source.map(IO.anonymizePath(rootDir, _)),
+        target = pc.target.map(IO.anonymizePath(rootDir, _))
+      ))
+    )
+
+    val encodedConfig: Json = encodeConfig(anonymizedConfig)
     // drop empty & null values recursively except all "info" fields
     val cleanEncodedConfig = encodedConfig.dropEmptyRecursivelyExcept(Seq("info", ".engines.entrypoint", ".engines.cmd"))
     // get config.info and *do* clean it

src/main/scala/io/viash/helpers/IO.scala

@@ -329,4 +329,29 @@ object IO extends Logging {
     val newPath = resolvePathWrtURI(path, uri)
     uri.resolve(newPath)
   }
+
+  /**
+    * Relativize a path w.r.t. a base path
+    * 
+    * If the path is not relative to the base path, the path is returned unchanged
+    *
+    * @param basePath The base path of the project
+    * @param path The path to relativize
+    * @return A relativized path or the original path if it was not relative to the base path
+    */
+  def anonymizePath(basePath: Option[Path], path: String): String = {    
+    val pathPath = Paths.get(path)
+
+    if (pathPath.isAbsolute()) {
+      val relative = basePath.map(_.relativize(pathPath).toString())
+
+      relative match {
+        case Some(rel) if rel.startsWith("..") => Paths.get("[anonymized]", pathPath.toFile().getName()).toString()
+        case Some(rel) => rel
+        case None => Paths.get("[anonymized]", pathPath.toFile().getName()).toString()
+      }
+    } else {
+      path
+    }
+  }
 }

src/main/scala/io/viash/project/ProjectConfig.scala

@@ -30,15 +30,18 @@ import io.viash.functionality.Author
 
 @description("A Viash project configuration file. It's name should be `_viash.yaml`.")
 @example(
-  """viash_version: 0.6.4
-    §source: src
-    §target: target
-    §config_mods: |
-    §  .engines[.type == 'docker'].target_registry := 'ghcr.io'
-    §  .engines[.type == 'docker'].target_organization := 'viash-io'
-    §  .engines[.type == 'docker'].namespace_separator := '/'
-    §  .engines[.type == 'docker'].target_image_source := 'https://github.com/viash-io/viash'
-    §""".stripMargin('§'), "yaml"
+  """viash_version: 0.9.0
+    |source: src
+    |target: target
+    |version: 1.0
+    |organization: viash-io
+    |links:
+    |  repository: 'https://github.com/viash-io/viash'
+    |  docker_registry: 'ghcr.io'
+    |config_mods: |
+    |  .runners[.type == 'nextflow'].directives.tag := '$id'
+    |  .runners[.type == 'nextflow'].config.script := 'includeConfig("configs/custom.config")'
+    |""".stripMargin, "yaml"
 )
 @since("Viash 0.6.4")
 @nameOverride("Project")

src/main/scala/io/viash/project/ProjectConfigLinks.scala

@@ -20,6 +20,13 @@ package io.viash.project
 import io.viash.schemas._
 
 @description("Links to external resources related to the project.")
+@example(
+  """repository: "https://github.com/viash-io/viash"
+    |docker_registry: "https://ghcr.io"
+    |homepage: "https://viash.io"
+    |documentation: "https://viash.io/reference/"
+    |issue_tracker: "https://github.com/viash-io/viash/issues"
+    |""".stripMargin, "yaml")
 @since("Viash 0.9.0")
 case class ProjectConfigLinks(
   @description("Source repository url.")

src/main/scala/io/viash/project/ProjectConfigReferences.scala

@@ -1,9 +1,36 @@
+/*
+ * Copyright (C) 2020  Data Intuitive
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package io.viash.project
 
 import io.viash.schemas._
 import io.viash.helpers.data_structures.OneOrMore
 
 @description("References to external resources related to the project.")
+@example(
+  """doi: 10.1000/xx.123456.789
+    |bibtex: |
+    |  @article{foo,
+    |    title={Foo},
+    |    author={Bar},
+    |    journal={Baz},
+    |    year={2024}
+    |  }
+    |""".stripMargin, "yaml")
 @since("Viash 0.9.0")
 case class ProjectConfigReferences(
   @description("One or multiple DOI reference(s) of the project.")

src/main/scala/io/viash/project/package.scala

@@ -17,19 +17,19 @@
 
 package io.viash
 
-import io.circe.{Decoder, Encoder, Json}
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
+import io.circe.{Decoder, Encoder}
 
 package object project {
   import io.viash.helpers.circe._
   import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
+  import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
 
-  implicit val encodeProjectConfigLinks: Encoder.AsObject[ProjectConfigLinks] = deriveConfiguredEncoder
+  implicit val encodeProjectConfigLinks: Encoder.AsObject[ProjectConfigLinks] = deriveConfiguredEncoderStrict
   implicit val decodeProjectConfigLinks: Decoder[ProjectConfigLinks] = deriveConfiguredDecoderFullChecks
 
-  implicit val encodeProjectConfigReferences: Encoder.AsObject[ProjectConfigReferences] = deriveConfiguredEncoder
+  implicit val encodeProjectConfigReferences: Encoder.AsObject[ProjectConfigReferences] = deriveConfiguredEncoderStrict
   implicit val decodeProjectConfigReferences: Decoder[ProjectConfigReferences] = deriveConfiguredDecoderFullChecks
 
-  implicit val encodeProjectConfig: Encoder.AsObject[ProjectConfig] = deriveConfiguredEncoder
+  implicit val encodeProjectConfig: Encoder.AsObject[ProjectConfig] = deriveConfiguredEncoderStrict
   implicit val decodeProjectConfig: Decoder[ProjectConfig] = deriveConfiguredDecoderFullChecks
 }

src/main/scala/io/viash/schemas/CollectedSchemas.scala

@@ -32,7 +32,7 @@ import monocle.function.Cons
 import io.viash.config.Config
 import io.viash.config.Info
 import io.viash.functionality.resources._
-import io.viash.project.ProjectConfig
+import io.viash.project.{ProjectConfig, ProjectConfigLinks, ProjectConfigReferences}
 import io.viash.helpers._
 import scala.collection.immutable.ListMap
 import io.viash.functionality.dependencies._
@@ -118,6 +118,8 @@ object CollectedSchemas {
   lazy val schemaClasses = List(
     getMembers[Config](),
     getMembers[ProjectConfig](),
+    getMembers[ProjectConfigLinks](),
+    getMembers[ProjectConfigReferences](),
     getMembers[Info](),
     getMembers[SysEnvTrait](),
 

src/test/scala/io/viash/e2e/build/DockerMeta.scala

@@ -58,11 +58,11 @@ class DockerMeta extends AnyFunSuite with BeforeAndAfterAll {
       val viashVersion = io.viash.Main.version
 
       val regexViashVersion = s"""viash_version: "${viashVersion}"""".r
-      val regexConfig = s"""config: "${configFile}"""".r
+      val regexConfig = """config: "\[anonymized\]/config.vsh.yaml"""".r
       val regexEngine = """engine: "docker"""".r
       val regexRunner = """runner: "docker"""".r
-      val regexExecutable = s"""executable: "$binDir/testbash"""".r
-      val regexOutput = s"""output: "$binDir"""".r
+      val regexExecutable = """executable: "\[anonymized\]/testbash"""".r
+      val regexOutput = """output: "\[anonymized\]/bin"""".r
       val regexNoRemoteGitRepo = "git_remote:".r
 
       assert(regexViashVersion.findFirstIn(metaStr).isDefined, stdout)
@@ -130,11 +130,11 @@ class DockerMeta extends AnyFunSuite with BeforeAndAfterAll {
       val viashVersion = io.viash.Main.version
 
       val regexViashVersion = s"""viash_version: "$viashVersion"""".r
-      val regexConfig = s"""config: "$configMetaFile"""".r
+      val regexConfig = """config: "\[anonymized\]/config.vsh.yaml"""".r
       val regexEngine = """engine: "docker"""".r
       val regexRunner = """runner: "docker"""".r
-      val regexExecutable = s"""executable: "$binDir/testbash"""".r
-      val regexOutput = s"""output: "$binDir"""".r
+      val regexExecutable = """executable: "\[anonymized\]/testbash"""".r
+      val regexOutput = """output: "\[anonymized\]/bin"""".r
       val regexRemoteGitRepo = s"""git_remote: "$fakeGitRepo"""".r
 
       assert(regexViashVersion.findFirstIn(metaStr).isDefined, stdout)
@@ -193,11 +193,11 @@ class DockerMeta extends AnyFunSuite with BeforeAndAfterAll {
       val viashVersion = io.viash.Main.version
 
       val regexViashVersion = s"""viash_version: "$viashVersion"""".r
-      val regexConfig = s"""config: "$configMetaFile"""".r
+      val regexConfig = """config: "\[anonymized\]/config.vsh.yaml"""".r
       val regexEngine = """engine: "docker"""".r
       val regexRunner = """runner: "docker"""".r
-      val regexExecutable = s"""executable: "$binDir/testbash"""".r
-      val regexOutput = s"""output: "$binDir"""".r
+      val regexExecutable = """executable: "\[anonymized\]/testbash"""".r
+      val regexOutput = """output: "\[anonymized\]/bin"""".r
       val regexRemoteGitRepo = """git_remote:"""".r
 
       assert(regexViashVersion.findFirstIn(metaStr).isDefined, stdout)

src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala

@@ -4,13 +4,14 @@ import io.viash._
 import io.viash.helpers.Logger
 
 import org.scalatest.funsuite.AnyFunSuite
+import java.nio.file.Paths
 
 class MainConfigViewSuite extends AnyFunSuite{
   Logger.UseColorOverride.value = Some(false)
   // path to namespace components
   private val configFile = getClass.getResource(s"/testbash/config.vsh.yaml").getPath
-
-
+  private val workingDirAnonymizing = getClass.getResource("/testns/").getPath()
+  private val configFileAnonymizing = getClass.getResource("/testns/src/ns_add/config.vsh.yaml").getPath
 
   test("viash config view local") {
     val testOutput = TestHelper.testMain(
@@ -32,4 +33,25 @@ class MainConfigViewSuite extends AnyFunSuite{
     assert(testOutput.stdout.contains("testbash"))
   }
 
+  test("viash config view wit anonymized paths") {
+    // Output is not anonymized when no project config is found, and that can't be found unless we pass the workingDir
+    val testOutput = TestHelper.testMain(
+      // workingDir = Some(Paths.get(workingDirAnonymizing)),
+      "config", "view",
+      configFileAnonymizing
+    )
+
+    assert(testOutput.stdout.contains(s"""config: "[anonymized]/config.vsh.yaml""""))
+  }
+
+  test("viash config view with relative anonymous paths") {
+    val testOutput = TestHelper.testMain(
+      workingDir = Some(Paths.get(workingDirAnonymizing)),
+      "config", "view",
+      configFileAnonymizing
+    )
+
+    assert(testOutput.stdout.contains("""config: "src/ns_add/config.vsh.yaml""""))
+  }
+
 }

src/test/scala/io/viash/helpers/IOTest.scala

@@ -6,6 +6,7 @@ import java.nio.file.{Files, Path}
 import java.net.URI
 import io.viash.helpers.{IO, Logger}
 import scala.util.Try
+import java.nio.file.Paths
 
 class IOTest extends AnyFunSuite with BeforeAndAfter {
   Logger.UseColorOverride.value = Some(false)
@@ -94,4 +95,39 @@ class IOTest extends AnyFunSuite with BeforeAndAfter {
     val result = Try(IO.resolveProjectPath("/test/test.txt", None))
     assert(result.isFailure)
   }
+
+  test("anonymizePath with a common path") {
+    val basePath = Some(Paths.get("/foo/bar"))
+    val path = "/foo/bar/baz/test.txt"
+    val result = IO.anonymizePath(basePath, path)
+    assert(result == "baz/test.txt")
+  }
+
+  test("anonymizePath with a different base path") {
+    val basePath = Some(Paths.get("/foo/bar"))
+    val path = "/foo/baz/test.txt"
+    val result = IO.anonymizePath(basePath, path)
+    assert(result == "[anonymized]/test.txt")
+  }
+
+  test("anynimizePath with a completely different base path") {
+    val basePath = Some(Paths.get("/foo/bar"))
+    val path = "/tmp/foo/bar/baz/test.txt"
+    val result = IO.anonymizePath(basePath, path)
+    assert(result == "[anonymized]/test.txt")
+  }
+
+  test("anonymizePath with a common path with a trailing slash") {
+    val basePath = Some(Paths.get("/foo/bar/"))
+    val path = "/foo/bar/baz/test.txt"
+    val result = IO.anonymizePath(basePath, path)
+    assert(result == "baz/test.txt")
+  }
+
+  test("anonymizePath without a base path") {
+    val basePath = None
+    val path = "/foo/bar/baz/test.txt"
+    val result = IO.anonymizePath(basePath, path)
+    assert(result == "[anonymized]/test.txt")
+  }
 }

src/test/scala/io/viash/project/ProjectTest.scala

@@ -45,5 +45,6 @@ class ProjectTest extends AnyFunSuite {
     val proj2 = ProjectConfig.findViashProject(rootPath.resolve("testns/src/ns_add"))
 
     assert(proj2 == proj)
+    assert(proj2.rootDir == Some(rootPath.resolve("testns")))
   }
 }