From 3380ff23bc1324183e723bc673f28eeb0d61e7ed Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 13 Jun 2024 12:03:08 +0200 Subject: [PATCH] Fix/nextflow executable build info (#720) * Decide whether the executable filename is the component name or 'main.nf' based on runner type and not mainScript type * Test build_info output Need to set workingDir, create a _viash.yaml file and need to remain in the same folder with the output folder to avoid anonymizing * Add changelog entry --- CHANGELOG.md | 4 ++ src/main/scala/io/viash/ViashBuild.scala | 2 +- src/main/scala/io/viash/ViashRun.scala | 2 +- src/main/scala/io/viash/ViashTest.scala | 2 +- .../scala/io/viash/config/ConfigMeta.scala | 9 ++- .../io/viash/e2e/build/NativeSuite.scala | 63 +++++++++++++++++++ 6 files changed, 76 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea7db760d..8947ffde7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ TODO add summary * `resources_test`: This field is removed again from the `_viash.yaml` as it was decided to impliment this temporary functionality using the `info` field (PR #711). +## BUG FIXES + +* `build_info`: Correctly set the `.build_info.executable` to `main.nf` when building a component with a Nextflow runner (PR #720). + * `vsh organization`: ViashHub repositories now use `vsh` as the default organization (PR #718). Instead of having to specify `repo: vsh/repo_name`, you can now just specify `repo: repo_name`, which is now also the prefered way. diff --git a/src/main/scala/io/viash/ViashBuild.scala b/src/main/scala/io/viash/ViashBuild.scala index 1ae81ff84..72b7aa032 100644 --- a/src/main/scala/io/viash/ViashBuild.scala +++ b/src/main/scala/io/viash/ViashBuild.scala @@ -42,7 +42,7 @@ object ViashBuild extends Logging { val exec_path = resources.mainScript.map(scr => Paths.get(output, scr.resourcePath).toString) // convert config to a yaml wrapped inside a PlainFile - val configYaml = ConfigMeta.toMetaFile(appliedConfig.config, Some(dir)) + val configYaml = ConfigMeta.toMetaFile(appliedConfig, Some(dir)) // write resources to output directory IO.writeResources(configYaml :: resources.resources, dir) diff --git a/src/main/scala/io/viash/ViashRun.scala b/src/main/scala/io/viash/ViashRun.scala index d525c4dc4..5ff5ab54a 100644 --- a/src/main/scala/io/viash/ViashRun.scala +++ b/src/main/scala/io/viash/ViashRun.scala @@ -42,7 +42,7 @@ object ViashRun extends Logging { var code = -1 try { // convert config to a yaml wrapped inside a PlainFile - val configYaml = ConfigMeta.toMetaFile(appliedConfig.config, Some(dir)) + val configYaml = ConfigMeta.toMetaFile(appliedConfig, Some(dir)) // write executable and resources to temporary directory IO.writeResources(configYaml :: resources.resources, dir) diff --git a/src/main/scala/io/viash/ViashTest.scala b/src/main/scala/io/viash/ViashTest.scala index b6ceb45dd..bd3e0ec57 100644 --- a/src/main/scala/io/viash/ViashTest.scala +++ b/src/main/scala/io/viash/ViashTest.scala @@ -280,7 +280,7 @@ object ViashTest extends Logging { dest = Some("test_executable"), text = resourcesOnlyTest.resources.head.text ) - val configYaml = ConfigMeta.toMetaFile(appliedConfig.config, Some(dir)) + val configYaml = ConfigMeta.toMetaFile(appliedConfig, Some(dir)) // assemble full resources list for test val confFinal = resourcesLens.set( diff --git a/src/main/scala/io/viash/config/ConfigMeta.scala b/src/main/scala/io/viash/config/ConfigMeta.scala index 20a87b028..63a56cedb 100644 --- a/src/main/scala/io/viash/config/ConfigMeta.scala +++ b/src/main/scala/io/viash/config/ConfigMeta.scala @@ -27,6 +27,7 @@ import io.circe.Json import io.circe.JsonObject import io.viash.config.resources.NextflowScript import io.viash.helpers.IO +import io.viash.runners.NextflowRunner object ConfigMeta { // create a yaml printer for writing the viash.yaml file @@ -71,14 +72,16 @@ object ConfigMeta { }) } - def toMetaFile(config: Config, buildDir: Option[Path]): PlainFile = { + def toMetaFile(appliedConfig: AppliedConfig, buildDir: Option[Path]): PlainFile = { + val config = appliedConfig.config + // get resources val placeholderMap = config.resources.filter(_.text.isDefined).map{ res => (res, "VIASH_PLACEHOLDER~" + res.filename + "~") }.toMap - val executableName = config.mainScript match { - case Some(n: NextflowScript) => "main.nf" + val executableName = appliedConfig.runner match { + case Some(_: NextflowRunner) => "main.nf" case _ => config.name } diff --git a/src/test/scala/io/viash/e2e/build/NativeSuite.scala b/src/test/scala/io/viash/e2e/build/NativeSuite.scala index f9c28419b..49ada6588 100644 --- a/src/test/scala/io/viash/e2e/build/NativeSuite.scala +++ b/src/test/scala/io/viash/e2e/build/NativeSuite.scala @@ -11,6 +11,7 @@ import io.viash.config.Config import scala.io.Source import io.viash.helpers.{IO, Exec, Logger} import io.viash.exceptions.ConfigParserException +import java.nio.file.Files class NativeSuite extends AnyFunSuite with BeforeAndAfterAll { Logger.UseColorOverride.value = Some(false) @@ -261,6 +262,68 @@ class NativeSuite extends AnyFunSuite with BeforeAndAfterAll { assert(testOutput.stderr.contains("Warning: no resources specified!")) } + test("Check data in build_info for executable runners") { + val newConfigFilePath = configDeriver.derive(Nil, "build_info_native") + Files.write(temporaryConfigFolder.resolve("_viash.yaml"), Array.emptyByteArray) + + val outputFolder = temporaryConfigFolder.resolve("output_build_info_native") + + val testOutput = TestHelper.testMain( + workingDir = Some(temporaryConfigFolder), + "build", + "-o", outputFolder.toString, + "--runner", "executable", + newConfigFilePath + ) + + val executable = outputFolder.resolve("testbash").toFile + val buildConfig = outputFolder.resolve(".config.vsh.yaml").toFile + + assert(executable.exists()) + assert(buildConfig.exists()) + + val buildConfigFile = Source.fromFile(buildConfig) + try { + val buildOutput = buildConfigFile.mkString + assert(buildOutput.contains("runner: \"executable\"")) + assert(buildOutput.contains("engine: \"native|docker|throwawayimage\"")) + assert(buildOutput.contains("executable: \"output_build_info_native/testbash\"")) + } finally { + buildConfigFile.close() + } + } + + test("Check data in build_info for nextflow runners") { + val newConfigFilePath = configDeriver.derive(Nil, "build_info_nextflow") + Files.write(temporaryConfigFolder.resolve("_viash.yaml"), Array.emptyByteArray) + + val outputFolder = temporaryConfigFolder.resolve("output_build_info_nextflow") + + val testOutput = TestHelper.testMain( + workingDir = Some(temporaryConfigFolder), + "build", + "-o", outputFolder.toString, + "--runner", "nextflow", + newConfigFilePath + ) + + val executable = outputFolder.resolve("main.nf").toFile + val buildConfig = outputFolder.resolve(".config.vsh.yaml").toFile + + assert(executable.exists()) + assert(buildConfig.exists()) + + val buildConfigFile = Source.fromFile(buildConfig) + try { + val buildOutput = buildConfigFile.mkString + assert(buildOutput.contains("runner: \"nextflow\"")) + assert(buildOutput.contains("engine: \"native|docker|throwawayimage\"")) + assert(buildOutput.contains("executable: \"output_build_info_nextflow/main.nf\"")) + } finally { + buildConfigFile.close() + } + } + override def afterAll(): Unit = { IO.deleteRecursively(temporaryFolder) IO.deleteRecursively(temporaryConfigFolder)