Skip to content

Commit

Permalink
[WX-1168] TES Log Paths (#7210)
Browse files Browse the repository at this point in the history
  • Loading branch information
THWiseman authored Aug 24, 2023
1 parent f64b3b9 commit aea7343
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,6 @@ trait TesJobCachingActorHelper extends StandardCachingActorHelper {
lazy val tesConfiguration: TesConfiguration = initializationData.tesConfiguration

lazy val runtimeAttributes = TesRuntimeAttributes(validatedRuntimeAttributes, jobDescriptor.runtimeAttributes, tesConfiguration)
override protected def nonStandardMetadata: Map[String, Any] = super.nonStandardMetadata ++ tesJobPaths.azureLogPathsForMetadata

}
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,10 @@ case class TesJobPaths private[tes] (override val workflowPaths: TesWorkflowPath
val callInputsRoot = callRoot.resolve("inputs")

/*
* tesTaskRoot: This is the root directory that TES will use for files related to this task.
* TES expects a path relative to the root of the storage container.
* We provide it to TES as a k/v pair where the key is "internal_path_prefix" and the value is the relative path string.
* tesTaskRoot: The Azure TES implementation allows us to specify a working directory that it should use for its own files.
* Once the task finishes, this directory will contain stderr.txt, stdout.txt, and some other misc files.
* Unlike other paths we provide, this one is expected as a path relative to the root of the storage container.
* We provide it as a k/v pair where the key is "internal_path_prefix" and the value is the relative path string.
* This is not a standard TES feature, but rather related to the Azure TES implementation that Terra uses.
* While passing it outside of terra won't do any harm, we could consider making this optional and/or configurable.
*/
Expand All @@ -42,6 +43,13 @@ case class TesJobPaths private[tes] (override val workflowPaths: TesWorkflowPath
case anyOtherPath: Path => anyOtherPath.pathAsString
}

// Like above: Nothing should rely on these files existing, since only the Azure TES implementation will actually create them.
// Used to send the Azure TES log paths to the frontend.
val azureLogPathsForMetadata : Map[String, Any] = Map(
"tes_stdout" -> taskFullPath./("stdout.txt").pathAsString,
"tes_stderr" -> taskFullPath./("stderr.txt").pathAsString
)

// Given an output path, return a path localized to the storage file system
def storageOutput(path: String): String = {
callExecutionRoot.resolve(path).toString
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ class TesJobPathsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers
File(s"/cromwell-executions/wf_hello/$id/call-hello").pathAsString
jobPaths.callExecutionDockerRoot.toString shouldBe
File(s"/cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString
jobPaths.azureLogPathsForMetadata shouldBe
Map(
"tes_stdout" -> File(s"local-cromwell-executions/wf_hello/$id/call-hello/tes_task/stdout.txt").pathAsString,
"tes_stderr" -> File(s"local-cromwell-executions/wf_hello/$id/call-hello/tes_task/stderr.txt").pathAsString
)

val jobKeySharded = BackendJobDescriptorKey(call, Option(0), 1)
val jobPathsSharded = TesJobPaths(jobKeySharded, wd, TesTestConfig.backendConfig)
Expand Down

0 comments on commit aea7343

Please sign in to comment.