Skip to content

Commit

Permalink
Fix/nextflow executable build info (#720)
Browse files Browse the repository at this point in the history
* Decide whether the executable filename is the component name or 'main.nf' based on runner type and not mainScript type

* Test build_info output

Need to set workingDir, create a _viash.yaml file and need to remain in the same folder with the output folder to avoid anonymizing

* Add changelog entry
  • Loading branch information
Grifs authored Jun 13, 2024
1 parent 4da8211 commit 3380ff2
Show file tree
Hide file tree
Showing 6 changed files with 76 additions and 6 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ TODO add summary

* `resources_test`: This field is removed again from the `_viash.yaml` as it was decided to impliment this temporary functionality using the `info` field (PR #711).

## BUG FIXES

* `build_info`: Correctly set the `.build_info.executable` to `main.nf` when building a component with a Nextflow runner (PR #720).

* `vsh organization`: ViashHub repositories now use `vsh` as the default organization (PR #718).
Instead of having to specify `repo: vsh/repo_name`, you can now just specify `repo: repo_name`, which is now also the prefered way.

Expand Down
2 changes: 1 addition & 1 deletion src/main/scala/io/viash/ViashBuild.scala
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ object ViashBuild extends Logging {
val exec_path = resources.mainScript.map(scr => Paths.get(output, scr.resourcePath).toString)

// convert config to a yaml wrapped inside a PlainFile
val configYaml = ConfigMeta.toMetaFile(appliedConfig.config, Some(dir))
val configYaml = ConfigMeta.toMetaFile(appliedConfig, Some(dir))

// write resources to output directory
IO.writeResources(configYaml :: resources.resources, dir)
Expand Down
2 changes: 1 addition & 1 deletion src/main/scala/io/viash/ViashRun.scala
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ object ViashRun extends Logging {
var code = -1
try {
// convert config to a yaml wrapped inside a PlainFile
val configYaml = ConfigMeta.toMetaFile(appliedConfig.config, Some(dir))
val configYaml = ConfigMeta.toMetaFile(appliedConfig, Some(dir))

// write executable and resources to temporary directory
IO.writeResources(configYaml :: resources.resources, dir)
Expand Down
2 changes: 1 addition & 1 deletion src/main/scala/io/viash/ViashTest.scala
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ object ViashTest extends Logging {
dest = Some("test_executable"),
text = resourcesOnlyTest.resources.head.text
)
val configYaml = ConfigMeta.toMetaFile(appliedConfig.config, Some(dir))
val configYaml = ConfigMeta.toMetaFile(appliedConfig, Some(dir))

// assemble full resources list for test
val confFinal = resourcesLens.set(
Expand Down
9 changes: 6 additions & 3 deletions src/main/scala/io/viash/config/ConfigMeta.scala
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import io.circe.Json
import io.circe.JsonObject
import io.viash.config.resources.NextflowScript
import io.viash.helpers.IO
import io.viash.runners.NextflowRunner

object ConfigMeta {
// create a yaml printer for writing the viash.yaml file
Expand Down Expand Up @@ -71,14 +72,16 @@ object ConfigMeta {
})
}

def toMetaFile(config: Config, buildDir: Option[Path]): PlainFile = {
def toMetaFile(appliedConfig: AppliedConfig, buildDir: Option[Path]): PlainFile = {
val config = appliedConfig.config

// get resources
val placeholderMap = config.resources.filter(_.text.isDefined).map{ res =>
(res, "VIASH_PLACEHOLDER~" + res.filename + "~")
}.toMap

val executableName = config.mainScript match {
case Some(n: NextflowScript) => "main.nf"
val executableName = appliedConfig.runner match {
case Some(_: NextflowRunner) => "main.nf"
case _ => config.name
}

Expand Down
63 changes: 63 additions & 0 deletions src/test/scala/io/viash/e2e/build/NativeSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import io.viash.config.Config
import scala.io.Source
import io.viash.helpers.{IO, Exec, Logger}
import io.viash.exceptions.ConfigParserException
import java.nio.file.Files

class NativeSuite extends AnyFunSuite with BeforeAndAfterAll {
Logger.UseColorOverride.value = Some(false)
Expand Down Expand Up @@ -261,6 +262,68 @@ class NativeSuite extends AnyFunSuite with BeforeAndAfterAll {
assert(testOutput.stderr.contains("Warning: no resources specified!"))
}

test("Check data in build_info for executable runners") {
val newConfigFilePath = configDeriver.derive(Nil, "build_info_native")
Files.write(temporaryConfigFolder.resolve("_viash.yaml"), Array.emptyByteArray)

val outputFolder = temporaryConfigFolder.resolve("output_build_info_native")

val testOutput = TestHelper.testMain(
workingDir = Some(temporaryConfigFolder),
"build",
"-o", outputFolder.toString,
"--runner", "executable",
newConfigFilePath
)

val executable = outputFolder.resolve("testbash").toFile
val buildConfig = outputFolder.resolve(".config.vsh.yaml").toFile

assert(executable.exists())
assert(buildConfig.exists())

val buildConfigFile = Source.fromFile(buildConfig)
try {
val buildOutput = buildConfigFile.mkString
assert(buildOutput.contains("runner: \"executable\""))
assert(buildOutput.contains("engine: \"native|docker|throwawayimage\""))
assert(buildOutput.contains("executable: \"output_build_info_native/testbash\""))
} finally {
buildConfigFile.close()
}
}

test("Check data in build_info for nextflow runners") {
val newConfigFilePath = configDeriver.derive(Nil, "build_info_nextflow")
Files.write(temporaryConfigFolder.resolve("_viash.yaml"), Array.emptyByteArray)

val outputFolder = temporaryConfigFolder.resolve("output_build_info_nextflow")

val testOutput = TestHelper.testMain(
workingDir = Some(temporaryConfigFolder),
"build",
"-o", outputFolder.toString,
"--runner", "nextflow",
newConfigFilePath
)

val executable = outputFolder.resolve("main.nf").toFile
val buildConfig = outputFolder.resolve(".config.vsh.yaml").toFile

assert(executable.exists())
assert(buildConfig.exists())

val buildConfigFile = Source.fromFile(buildConfig)
try {
val buildOutput = buildConfigFile.mkString
assert(buildOutput.contains("runner: \"nextflow\""))
assert(buildOutput.contains("engine: \"native|docker|throwawayimage\""))
assert(buildOutput.contains("executable: \"output_build_info_nextflow/main.nf\""))
} finally {
buildConfigFile.close()
}
}

override def afterAll(): Unit = {
IO.deleteRecursively(temporaryFolder)
IO.deleteRecursively(temporaryConfigFolder)
Expand Down

0 comments on commit 3380ff2

Please sign in to comment.