From 6e8399bebca8d2555cd80daa7e1b7307cc9f1b09 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Mon, 18 Mar 2024 12:21:25 -0400 Subject: [PATCH 01/23] Implement `returnCodes` runtime attribute --- .../BackendWorkflowInitializationActor.scala | 12 +- .../StandardAsyncExecutionActor.scala | 15 ++- ...ardValidatedRuntimeAttributesBuilder.scala | 1 + .../backend/validation/ReturnCodes.scala | 44 +++++++ .../validation/ReturnCodesValidation.scala | 71 +++++++++++ .../RuntimeAttributesValidation.scala | 6 +- ...ckendWorkflowInitializationActorSpec.scala | 120 +++++++++++++++++- ...alidatedRuntimeAttributesBuilderSpec.scala | 28 ++++ .../RuntimeAttributesValidationSpec.scala | 67 ++++++++++ .../failing_return_code.test | 11 ++ .../failing_return_code.wdl | 20 +++ .../returnCodes/return_codes.wdl | 69 ++++++++++ .../standardTestCases/return_codes.test | 11 ++ .../impl/aws/AwsBatchRuntimeAttributes.scala | 10 ++ .../backend/impl/aws/AwsBatchJobSpec.scala | 3 +- .../aws/AwsBatchRuntimeAttributesSpec.scala | 58 ++++++++- .../models/GcpBatchRuntimeAttributes.scala | 9 ++ .../GcpBatchRuntimeAttributesSpec.scala | 11 +- .../PipelinesApiRuntimeAttributes.scala | 9 ++ .../PipelinesApiRuntimeAttributesSpec.scala | 41 +++++- .../impl/tes/TesRuntimeAttributes.scala | 10 ++ .../impl/tes/TesRuntimeAttributesSpec.scala | 45 ++++++- .../backend/impl/tes/TesTaskSpec.scala | 3 +- .../main/scala/wom/RuntimeAttributes.scala | 3 + 24 files changed, 665 insertions(+), 12 deletions(-) create mode 100644 backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala create mode 100644 backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala create mode 100644 centaur/src/main/resources/standardTestCases/failing_return_code.test create mode 100644 centaur/src/main/resources/standardTestCases/failing_return_code/failing_return_code.wdl create mode 100644 centaur/src/main/resources/standardTestCases/returnCodes/return_codes.wdl create mode 100644 centaur/src/main/resources/standardTestCases/return_codes.test diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala index 9f59a451478..8435a1019d2 100644 --- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala @@ -11,7 +11,7 @@ import common.validation.Validation._ import cromwell.backend.BackendLifecycleActor._ import cromwell.backend.BackendWorkflowInitializationActor._ import cromwell.backend.async.{RuntimeAttributeValidationFailure, RuntimeAttributeValidationFailures} -import cromwell.backend.validation.ContinueOnReturnCodeValidation +import cromwell.backend.validation.{ContinueOnReturnCodeValidation, ReturnCodesValidation} import cromwell.core._ import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import cromwell.services.metadata.MetadataService.PutMetadataAction @@ -134,6 +134,16 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w .default(configurationDescriptor.backendRuntimeAttributesConfig) .validateOptionalWomValue(womExpressionMaybe) + /** + * This predicate is only appropriate for validation during workflow initialization. The logic does not differentiate + * between evaluation failures due to missing call inputs or evaluation failures due to malformed expressions, and will + * return `true` in both cases. + */ + protected def returnCodesPredicate(valueRequired: Boolean)(womExpressionMaybe: Option[WomValue]): Boolean = + ReturnCodesValidation + .default(configurationDescriptor.backendRuntimeAttributesConfig) + .validateOptionalWomValue(womExpressionMaybe) + protected def runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean] // FIXME: If a workflow executes jobs using multiple backends, diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala index 1b183a67646..27085121c91 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala @@ -755,6 +755,14 @@ trait StandardAsyncExecutionActor lazy val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) + /** + * Returns the behavior for continuing on the return code, obtained by converting `returnCodeContents` to an Int. + * + * @return the behavior for continuing on the return code. + */ + lazy val returnCode: ReturnCodes = + RuntimeAttributesValidation.extract(ReturnCodesValidation.instance, validatedRuntimeAttributes) + /** * Returns the max number of times that a failed job should be retried, obtained by converting `maxRetries` to an Int. */ @@ -1388,7 +1396,8 @@ trait StandardAsyncExecutionActor ) ) retryElseFail(executionHandle) - case Success(returnCodeAsInt) if continueOnReturnCode.continueFor(returnCodeAsInt) => + case Success(returnCodeAsInt) + if continueOnReturnCode.continueFor(returnCodeAsInt) || returnCode.continueFor(returnCodeAsInt) => handleExecutionSuccess(status, oldHandle, returnCodeAsInt) // It's important that we check retryWithMoreMemory case before isAbort. RC could be 137 in either case; // if it was caused by OOM killer, want to handle as OOM and not job abort. @@ -1422,7 +1431,9 @@ trait StandardAsyncExecutionActor } else { tryReturnCodeAsInt match { case Success(returnCodeAsInt) - if outOfMemoryDetected && memoryRetryRequested && !continueOnReturnCode.continueFor(returnCodeAsInt) => + if outOfMemoryDetected && memoryRetryRequested && (!continueOnReturnCode.continueFor( + returnCodeAsInt) + || !returnCode.continueFor(returnCodeAsInt)) => val executionHandle = Future.successful( FailedNonRetryableExecutionHandle( RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log), diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala index 77f890b4a19..cf8c672b6b8 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala @@ -32,6 +32,7 @@ object StandardValidatedRuntimeAttributesBuilder { def default(backendRuntimeConfig: Option[Config]): StandardValidatedRuntimeAttributesBuilder = { val required = Seq( ContinueOnReturnCodeValidation.default(backendRuntimeConfig), + ReturnCodesValidation.default(backendRuntimeConfig), FailOnStderrValidation.default(backendRuntimeConfig), MaxRetriesValidation.default(backendRuntimeConfig) ) diff --git a/backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala b/backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala new file mode 100644 index 00000000000..d97a5aff6c8 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala @@ -0,0 +1,44 @@ +package cromwell.backend.validation + +import wom.types._ + +object ReturnCodes { + val validWdlTypes = Set[WomType](WomArrayType(WomIntegerType), WomStringType, WomIntegerType) +} + +/** + * Decides if a call/job continues upon a specific return code. + */ +sealed trait ReturnCodes { + + /** + * Returns true if the call is a success based on the return code. + * + * @param returnCode Return code from the process / script. + * @return True if the call is a success. + */ + final def continueFor(returnCode: Int): Boolean = + this match { + case ReturnCodesString(continue) => continue.equals("*") || returnCode == 0 + case ReturnCodesSet(returnCodes) => returnCodes.contains(returnCode) + } +} + +/** + * Continues based on a string, if "*" all return codes continue. + * @param returnCode If "*", all return codes are valid for continuing. + */ +case class ReturnCodesString(returnCode: String) extends ReturnCodes { + override def toString = returnCode +} + +/** + * Continues only if the call/job return code is found in returnCodes. + * @param returnCodes Inclusive set of return codes that specify a job success. + */ +case class ReturnCodesSet(returnCodes: Set[Int]) extends ReturnCodes { + override def toString = returnCodes match { + case single if single.size == 1 => returnCodes.head.toString + case multiple => s"[${multiple.mkString(",")}]" + } +} diff --git a/backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala new file mode 100644 index 00000000000..755e9a658fa --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala @@ -0,0 +1,71 @@ +package cromwell.backend.validation + +import cats.data.Validated.{Invalid, Valid} +import cats.implicits.{catsSyntaxValidatedId, toTraverseOps} +import com.typesafe.config.Config +import common.validation.ErrorOr.ErrorOr +import cromwell.backend.validation.RuntimeAttributesValidation.validateInt +import wom.RuntimeAttributesKeys +import wom.RuntimeAttributesKeys._ +import wom.types.{WomArrayType, WomIntegerType, WomStringType, WomType} +import wom.values.{WomArray, WomInteger, WomString, WomValue} + +import scala.util.Try + +/** + * Validates the "returnCodes" runtime attribute as an Integer, returning the value as an `Int`, or as + * an array of Integers, returning the value as Array[Int], or "*", returning the value as a String. + * + * `default` a hardcoded default WomValue for returnCodes. + * + * `configDefaultWdlValue` returns the value of the attribute as specified by the + * reference.conf file, coerced into a WomValue. + */ +object ReturnCodesValidation { + lazy val instance: RuntimeAttributesValidation[ReturnCodes] = new ReturnCodesValidation(ReturnCodesKey) + lazy val optional: OptionalRuntimeAttributesValidation[ReturnCodes] = instance.optional + def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ReturnCodes] = + instance.withDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) + + def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = + instance.configDefaultWomValue(runtimeConfig) + def configDefaultWomValue(config: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(config) +} + +class ReturnCodesValidation(attributeName: String) extends RuntimeAttributesValidation[ReturnCodes] { + + override def key: String = RuntimeAttributesKeys.ReturnCodesKey + + override def coercion: Set[WomType] = ReturnCodes.validWdlTypes + + override def validateValue: PartialFunction[WomValue, ErrorOr[ReturnCodes]] = { + case WomString(value) if value.equals("*") => + ReturnCodesString(value).validNel + case WomString(value) if Try(value.toInt).isSuccess => + ReturnCodesSet(Set(value.toInt)).validNel + case WomInteger(value) => + ReturnCodesSet(Set(value)).validNel + case value @ WomArray(_, seq) => + val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int] + errorOrInts match { + case Valid(ints) => ReturnCodesSet(ints.toSet).validNel + case Invalid(_) => invalidValueFailure(value) + } + } + + override def validateExpression: PartialFunction[WomValue, Boolean] = { + case WomString(value) if Try(value.toInt).isSuccess => true + case WomInteger(_) => true + case WomString(value) if value.equals("*") => true + case WomArray(WomArrayType(WomStringType), elements) => + elements forall { value => + Try(value.valueString.toInt).isSuccess + } + case WomArray(WomArrayType(WomIntegerType), _) => true + } + + override protected def missingValueMessage: String = s"Expecting $key" + + " runtime attribute to be either a String '*' or an Array[Int]" + + override def usedInCallCaching: Boolean = true +} diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala index 5fa52dac53a..54b864f4471 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala @@ -37,6 +37,9 @@ object RuntimeAttributesValidation { ): ErrorOr[ContinueOnReturnCode] = validateWithValidation(value, ContinueOnReturnCodeValidation.instance, onMissingKey) + def validateReturnCodes(value: Option[WomValue], onMissingKey: => ErrorOr[ReturnCodes]): ErrorOr[ReturnCodes] = + validateWithValidation(value, ReturnCodesValidation.instance, onMissingKey) + def validateMemory(value: Option[WomValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = validateWithValidation(value, MemoryValidation.instance(), onMissingKey) @@ -372,7 +375,8 @@ trait RuntimeAttributesValidation[ValidatedType] { case Success(womValue) => validateExpression.applyOrElse(womValue, (_: Any) => false) case Failure(_) => true // If we can't evaluate it, we'll let it pass for now... } - case Some(womValue) => validateExpression.applyOrElse(womValue, (_: Any) => false) + case Some(womValue) => + validateExpression.applyOrElse(womValue, (_: Any) => false) } def validateOptionalWomExpression(womExpressionMaybe: Option[WomExpression]): Boolean = diff --git a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala index 4fc1efd7381..e14b4737687 100644 --- a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala @@ -5,7 +5,14 @@ import _root_.wdl.draft2.model.types._ import akka.actor.ActorRef import akka.testkit.TestActorRef import com.typesafe.config.{Config, ConfigFactory} -import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet, ContinueOnReturnCodeValidation} +import cromwell.backend.validation.{ + ContinueOnReturnCodeFlag, + ContinueOnReturnCodeSet, + ContinueOnReturnCodeValidation, + ReturnCodesSet, + ReturnCodesString, + ReturnCodesValidation +} import cromwell.core.{TestKitSuite, WorkflowOptions} import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers @@ -37,11 +44,16 @@ class BackendWorkflowInitializationActorSpec val testContinueOnReturnCode: Option[WomValue] => Boolean = testPredicateBackendWorkflowInitializationActor.continueOnReturnCodePredicate(valueRequired = false) + val testReturnCodes: Option[WomValue] => Boolean = + testPredicateBackendWorkflowInitializationActor.returnCodesPredicate(valueRequired = false) + val optionalConfig: Option[Config] = Option(TestConfig.optionalRuntimeConfig) - it should "continueOnReturnCodePredicate" in { + it should "continueOnReturnCodePredicate and returnCodePredicate" in { testContinueOnReturnCode(None) should be(true) + testReturnCodes(None) should be(true) ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(None) should be(true) + ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(None) should be(true) val booleanRows = Table( "value", @@ -62,6 +74,11 @@ class BackendWorkflowInitializationActorSpec "read_int(\"bad file\")" ) + val starRow = Table( + "value", + "*" + ) + val invalidWdlValueRows = Table( "womValue", WomString(""), @@ -115,15 +132,26 @@ class BackendWorkflowInitializationActorSpec val womValue = WomInteger(value) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) + testReturnCodes(Option(womValue)) should be(result) ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) + ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = ContinueOnReturnCodeValidation .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) + + val returnCodesValid = + ReturnCodesValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) + returnCodesValid.isValid should be(result) + returnCodesValid.toEither.toOption.get should be(ReturnCodesSet(Set(value))) } forAll(integerRows) { value => @@ -133,12 +161,25 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) + + testReturnCodes(Option(womValue)) should be(result) + ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) + val valid = ContinueOnReturnCodeValidation .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) + + val returnCodesValid = + ReturnCodesValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) + returnCodesValid.isValid should be(result) + returnCodesValid.toEither.toOption.get should be(ReturnCodesSet(Set(value))) } forAll(integerRows) { value => @@ -148,6 +189,12 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) + + testReturnCodes(Option(womValue)) should be(result) + ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) + // NOTE: expressions are never valid to validate } @@ -158,12 +205,25 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) + + testReturnCodes(Option(womValue)) should be(result) + ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) + val valid = ContinueOnReturnCodeValidation .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) + + val returnCodesValid = + ReturnCodesValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) + returnCodesValid.isValid should be(result) + returnCodesValid.toEither.toOption.get should be(ReturnCodesSet(Set(value))) } forAll(integerRows) { value => @@ -173,12 +233,25 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) + + testReturnCodes(Option(womValue)) should be(result) + ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) + val valid = ContinueOnReturnCodeValidation .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) + + val returnCodesValid = + ReturnCodesValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) + returnCodesValid.isValid should be(result) + returnCodesValid.toEither.toOption.get should be(ReturnCodesSet(Set(value))) } forAll(integerRows) { value => @@ -188,6 +261,11 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) + + testReturnCodes(Option(womValue)) should be(result) + ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) // NOTE: expressions are never valid to validate } @@ -198,15 +276,41 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) + + testReturnCodes(Option(womValue)) should be(result) + ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) // NOTE: expressions are never valid to validate } + forAll(starRow) { value => + val womValue = WomString(value) + val result = true + testReturnCodes(Option(womValue)) should be(result) + ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) + val valid = + ReturnCodesValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) + valid.isValid should be(result) + valid.toEither.toOption.get should be(ReturnCodesString(value)) + } + forAll(invalidWdlValueRows) { womValue => val result = false testContinueOnReturnCode(Option(womValue)) should be(result) ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) + + testReturnCodes(Option(womValue)) should be(result) + ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) + val valid = ContinueOnReturnCodeValidation .default(optionalConfig) @@ -215,6 +319,15 @@ class BackendWorkflowInitializationActorSpec valid.toEither.swap.toOption.get.toList should contain theSameElementsAs List( "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) + + val returnCodesValid = + ReturnCodesValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) + returnCodesValid.isValid should be(result) + returnCodesValid.toEither.swap.toOption.get.toList should contain theSameElementsAs List( + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + ) } } @@ -247,4 +360,7 @@ class TestPredicateBackendWorkflowInitializationActor extends BackendWorkflowIni override def continueOnReturnCodePredicate(valueRequired: Boolean)(wdlExpressionMaybe: Option[WomValue]): Boolean = super.continueOnReturnCodePredicate(valueRequired)(wdlExpressionMaybe) + + override def returnCodesPredicate(valueRequired: Boolean)(wdlExpressionMaybe: Option[WomValue]): Boolean = + super.returnCodesPredicate(valueRequired)(wdlExpressionMaybe) } diff --git a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala index bd3024eda9a..5c3573c5b0c 100644 --- a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala @@ -153,6 +153,34 @@ class StandardValidatedRuntimeAttributesBuilderSpec ) } + "validate a valid returnCode entry" in { + val runtimeAttributes = Map("returnCodes" -> WomInteger(1)) + val expectedRuntimeAttributes = + defaultRuntimeAttributes + (ReturnCodesKey -> ReturnCodesSet(Set(1))) + assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + } + + "fail to validate an invalid returnCode entry" in { + val runtimeAttributes = Map("returnCodes" -> WomString("value")) + assertRuntimeAttributesFailedCreation( + runtimeAttributes, + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + ) + } + + "use workflow options as default if returnCode key is missing" in { + val expectedRuntimeAttributes = defaultRuntimeAttributes + + (ReturnCodesKey -> ReturnCodesSet(Set(1, 2))) + val workflowOptions = workflowOptionsWithDefaultRuntimeAttributes( + Map(ReturnCodesKey -> JsArray(Vector(JsNumber(1), JsNumber(2)))) + ) + val runtimeAttributes = Map.empty[String, WomValue] + assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, + expectedRuntimeAttributes, + workflowOptions = workflowOptions + ) + } + } val defaultLogger: Logger = LoggerFactory.getLogger(classOf[StandardValidatedRuntimeAttributesBuilderSpec]) diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala index 8f692d6486a..862aa001d99 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala @@ -228,6 +228,73 @@ class RuntimeAttributesValidationSpec } } + "return success when tries to validate a valid returnCodes string entry" in { + val returnCodesValue = Some(WomString("*")) + val result = RuntimeAttributesValidation.validateReturnCodes( + returnCodesValue, + "Failed to get return code mandatory key from runtime attributes".invalidNel + ) + result match { + case Valid(x) => assert(x == ReturnCodesString("*")) + case Invalid(e) => fail(e.toList.mkString(" ")) + } + } + + "return success when tries to validate a returnCodes int entry" in { + val returnCodesValue = Some(WomInteger(12)) + val result = RuntimeAttributesValidation.validateReturnCodes( + returnCodesValue, + "Failed to get returnCodes mandatory key from runtime attributes".invalidNel + ) + result match { + case Valid(x) => assert(x == ReturnCodesSet(Set(12))) + case Invalid(e) => fail(e.toList.mkString(" ")) + } + } + + "return success when there is a valid integer array in returnCodes runtime attribute" in { + val returnCodesValue = Some(WomArray(WomArrayType(WomIntegerType), Seq(WomInteger(1), WomInteger(2)))) + val result = RuntimeAttributesValidation.validateReturnCodes( + returnCodesValue, + "Failed to get returnCode mandatory key from runtime attributes".invalidNel + ) + result match { + case Valid(x) => assert(x == ReturnCodesSet(Set(1, 2))) + case Invalid(e) => fail(e.toList.mkString(" ")) + } + } + + "return failure when there is an invalid array in returnCodes runtime attribute" in { + val returnCodesValue = + Some(WomArray(WomArrayType(WomStringType), Seq(WomString("one"), WomString("two")))) + val result = RuntimeAttributesValidation.validateReturnCodes( + returnCodesValue, + "Failed to get returnCodes mandatory key from runtime attributes".invalidNel + ) + result match { + case Valid(_) => fail("A failure was expected.") + case Invalid(e) => + assert( + e.head == "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + ) + } + } + + "return failure when there is an invalid returnCodes runtime attribute defined" in { + val returnCodesValue = Some(WomString("yes")) + val result = RuntimeAttributesValidation.validateReturnCodes( + returnCodesValue, + "Failed to get returnCodes mandatory key from runtime attributes".invalidNel + ) + result match { + case Valid(_) => fail("A failure was expected.") + case Invalid(e) => + assert( + e.head == "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + ) + } + } + "return success when tries to validate a valid Integer memory entry" in { val expectedGb = 1 val memoryValue = Some(WomInteger(1 << 30)) diff --git a/centaur/src/main/resources/standardTestCases/failing_return_code.test b/centaur/src/main/resources/standardTestCases/failing_return_code.test new file mode 100644 index 00000000000..a5a9196b8b1 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/failing_return_code.test @@ -0,0 +1,11 @@ +name: failing_return_code +testFormat: workflowfailure + +files { + workflow: failing_return_code/failing_return_code.wdl +} + +metadata { + workflowName: FailingReturnCode + status: Failed +} \ No newline at end of file diff --git a/centaur/src/main/resources/standardTestCases/failing_return_code/failing_return_code.wdl b/centaur/src/main/resources/standardTestCases/failing_return_code/failing_return_code.wdl new file mode 100644 index 00000000000..f393698d2b7 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/failing_return_code/failing_return_code.wdl @@ -0,0 +1,20 @@ +version development-1.1 + +workflow FailingReturnCode { + call FailingReturnCodeSet +} + +task FailingReturnCodeSet { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + returnCodes: [0, 5, 10] + docker: "ubuntu:latest" + } +} diff --git a/centaur/src/main/resources/standardTestCases/returnCodes/return_codes.wdl b/centaur/src/main/resources/standardTestCases/returnCodes/return_codes.wdl new file mode 100644 index 00000000000..d760a5c1f81 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/returnCodes/return_codes.wdl @@ -0,0 +1,69 @@ +version development-1.1 + +workflow ReturnCodeValidation { + call ReturnCodeSet1 + call ReturnCodeSet2 + call ReturnCodeSet3 + call ReturnCodeString +} + +task ReturnCodeSet1 { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1] + } +} + +task ReturnCodeSet2 { + meta { + volatile: true + } + + command <<< + exit 200 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1, 123, 200] + } +} + +task ReturnCodeSet3 { + meta { + volatile: true + } + + command <<< + exit 10 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: 10 + } +} + + +task ReturnCodeString { + meta { + volatile: true + } + + command <<< + exit 500 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: "*" + } +} \ No newline at end of file diff --git a/centaur/src/main/resources/standardTestCases/return_codes.test b/centaur/src/main/resources/standardTestCases/return_codes.test new file mode 100644 index 00000000000..7528064f4c8 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/return_codes.test @@ -0,0 +1,11 @@ +name: return_codes +testFormat: workflowsuccess + +files { + workflow: return_codes/return_codes.wdl +} + +metadata { + workflowName: ReturnCodeValidation + status: Succeeded +} diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala index 447f48ae2b9..de1184f1afd 100755 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala @@ -57,6 +57,7 @@ import scala.util.matching.Regex * @param queueArn the arn of the AWS Batch queue that the job will be submitted to * @param failOnStderr should the job fail if something is logged to `stderr` * @param continueOnReturnCode decides if a job continues on receiving a specific return code + * @param returnCodes decides if a job continues on receiving a specific return code * @param noAddress is there no address * @param scriptS3BucketName the s3 bucket where the execution command or script will be written and, from there, fetched into the container and executed * @param fileSystem the filesystem type, default is "s3" @@ -69,6 +70,7 @@ case class AwsBatchRuntimeAttributes(cpu: Int Refined Positive, queueArn: String, failOnStderr: Boolean, continueOnReturnCode: ContinueOnReturnCode, + returnCodes: ReturnCodes, noAddress: Boolean, scriptS3BucketName: String, fileSystem: String = "s3" @@ -102,6 +104,9 @@ object AwsBatchRuntimeAttributes { private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig) + private def returnCodesValidation(runtimeConfig: Option[Config]) = + ReturnCodesValidation.default(runtimeConfig) + private def disksValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Seq[AwsBatchVolume]] = DisksValidation .withDefault(DisksValidation.configDefaultWomValue(runtimeConfig) getOrElse DisksDefaultValue) @@ -188,6 +193,10 @@ object AwsBatchRuntimeAttributes { continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) + val returnCodes: ReturnCodes = RuntimeAttributesValidation.extract( + returnCodesValidation(runtimeAttrsConfig), + validatedRuntimeAttributes + ) val noAddress: Boolean = RuntimeAttributesValidation.extract(noAddressValidation(runtimeAttrsConfig), validatedRuntimeAttributes) val scriptS3BucketName = fileSystem match { @@ -207,6 +216,7 @@ object AwsBatchRuntimeAttributes { queueArn, failOnStderr, continueOnReturnCode, + returnCodes, noAddress, scriptS3BucketName, fileSystem diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala index 32d0cfab6bb..edf4f7dd9b7 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala @@ -34,7 +34,7 @@ package cromwell.backend.impl.aws import common.collections.EnhancedCollections._ import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.backend.BackendSpec._ -import cromwell.backend.validation.ContinueOnReturnCodeFlag +import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ReturnCodesSet} import cromwell.core.TestKitSuite import cromwell.util.SampleWdl import eu.timepit.refined.api.Refined @@ -110,6 +110,7 @@ class AwsBatchJobSpec extends TestKitSuite with AnyFlatSpecLike with Matchers wi queueArn = "arn:aws:batch:us-east-1:123456789:job-queue/default-gwf-core", failOnStderr = true, continueOnReturnCode = ContinueOnReturnCodeFlag(false), + returnCodes = ReturnCodesSet(Set(0)), noAddress = false, scriptS3BucketName = "script-bucket", fileSystem = "s3" diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala index 957126da9ad..728d30f78f3 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala @@ -35,7 +35,12 @@ import cats.data.NonEmptyList import common.assertion.CromwellTimeoutSpec import cromwell.backend.RuntimeAttributeDefinition import cromwell.backend.impl.aws.io.{AwsBatchVolume, AwsBatchWorkingDisk} -import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet} +import cromwell.backend.validation.{ + ContinueOnReturnCodeFlag, + ContinueOnReturnCodeSet, + ReturnCodesSet, + ReturnCodesString +} import cromwell.core.WorkflowOptions import eu.timepit.refined.numeric.Positive import eu.timepit.refined.refineMV @@ -68,6 +73,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "arn:aws:batch:us-east-1:111222333444:job-queue/job-queue", false, ContinueOnReturnCodeSet(Set(0)), + ReturnCodesSet(Set(0)), false, "my-stuff" ) @@ -81,6 +87,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "arn:aws:batch:us-east-1:111222333444:job-queue/job-queue", false, ContinueOnReturnCodeSet(Set(0)), + ReturnCodesSet(Set(0)), false, "", "local" @@ -206,6 +213,55 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout ) } + "validate a valid returnCodes integer entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), + "scriptBucketName" -> WomString("my-stuff"), + "returnCodes" -> WomInteger(1) + ) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1))) + assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + } + + "validate a valid returnCodes String entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), + "scriptBucketName" -> WomString("my-stuff"), + "returnCodes" -> WomString("*") + ) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesString("*")) + assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + } + + "validate a valid returnCodes array entry" in { + val runtimeAttributes = Map( + "docker" -> WomString("ubuntu:latest"), + "scriptBucketName" -> WomString("my-stuff"), + "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) + ) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1, 2))) + assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + } + + "coerce then validate a valid returnCodes array entry" in { + val runtimeAttributes = Map( + "docker" -> WomString("ubuntu:latest"), + "scriptBucketName" -> WomString("my-stuff"), + "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) + ) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1, 2))) + assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + } + + "fail to validate an invalid returnCode entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), + "scriptBucketName" -> WomString("my-stuff"), + "returnCodes" -> WomString("value") + ) + assertAwsBatchRuntimeAttributesFailedCreation( + runtimeAttributes, + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + ) + } + "validate a valid cpu entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "scriptBucketName" -> WomString("my-stuff"), "cpu" -> WomInteger(2)) diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala index ef9e2389d4b..82a3371f207 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala @@ -47,6 +47,7 @@ final case class GcpBatchRuntimeAttributes(cpu: Int Refined Positive, dockerImage: String, failOnStderr: Boolean, continueOnReturnCode: ContinueOnReturnCode, + returnCodes: ReturnCodes, noAddress: Boolean, useDockerImageCache: Option[Boolean], checkpointFilename: Option[String] @@ -114,6 +115,9 @@ object GcpBatchRuntimeAttributes { private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig) + private def returnCodesValidation(runtimeConfig: Option[Config]) = + ReturnCodesValidation.default(runtimeConfig) + private def disksValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Seq[GcpBatchAttachedDisk]] = DisksValidation .withDefault(DisksValidation.configDefaultWomValue(runtimeConfig) getOrElse DisksDefaultValue) @@ -211,6 +215,10 @@ object GcpBatchRuntimeAttributes { continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) + val returnCodes: ReturnCodes = RuntimeAttributesValidation.extract( + returnCodesValidation(runtimeAttrsConfig), + validatedRuntimeAttributes + ) val noAddress: Boolean = RuntimeAttributesValidation.extract(noAddressValidation(runtimeAttrsConfig), validatedRuntimeAttributes) val zones: Vector[String] = RuntimeAttributesValidation.extract(ZonesValidation, validatedRuntimeAttributes) @@ -239,6 +247,7 @@ object GcpBatchRuntimeAttributes { docker, failOnStderr, continueOnReturnCode, + returnCodes, noAddress, useDockerImageCache, checkpointFileName diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala index beb9b865c1b..108dd3e8173 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala @@ -3,7 +3,7 @@ package cromwell.backend.google.batch.models import cats.data.NonEmptyList import cromwell.backend.RuntimeAttributeDefinition import cromwell.backend.google.batch.models.GcpBatchTestConfig._ -import cromwell.backend.validation.ContinueOnReturnCodeSet +import cromwell.backend.validation.{ContinueOnReturnCodeSet, ReturnCodesSet} //import cromwell.backend.google.batch.io.{DiskType, GcpBatchAttachedDisk} import cromwell.backend.google.batch.io.{DiskType, GcpBatchWorkingDisk} import cromwell.core.WorkflowOptions @@ -75,6 +75,14 @@ final class GcpBatchRuntimeAttributesSpec ) } + "fail to validate an invalid returnCodes entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("value")) + assertBatchRuntimeAttributesFailedCreation( + runtimeAttributes, + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + ) + } + "validate a valid cpu entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "cpu" -> WomInteger(2)) val expectedRuntimeAttributes = expectedDefaults.copy(cpu = refineMV(2)) @@ -275,6 +283,7 @@ trait GcpBatchRuntimeAttributesSpecsMixin { dockerImage = "ubuntu:latest", failOnStderr = false, continueOnReturnCode = ContinueOnReturnCodeSet(Set(0)), + returnCodes = ReturnCodesSet(Set(0)), noAddress = false, useDockerImageCache = None, checkpointFilename = None diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala index a472007b611..183dec9311a 100644 --- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala +++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala @@ -50,6 +50,7 @@ final case class PipelinesApiRuntimeAttributes(cpu: Int Refined Positive, dockerImage: String, failOnStderr: Boolean, continueOnReturnCode: ContinueOnReturnCode, + returnCodes: ReturnCodes, noAddress: Boolean, googleLegacyMachineSelection: Boolean, useDockerImageCache: Option[Boolean], @@ -116,6 +117,9 @@ object PipelinesApiRuntimeAttributes { private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig) + private def returnCodesValidation(runtimeConfig: Option[Config]) = + ReturnCodesValidation.default(runtimeConfig) + private def disksValidation( runtimeConfig: Option[Config] ): RuntimeAttributesValidation[Seq[PipelinesApiAttachedDisk]] = DisksValidation @@ -223,6 +227,10 @@ object PipelinesApiRuntimeAttributes { continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) + val returnCodes: ReturnCodes = RuntimeAttributesValidation.extract( + returnCodesValidation(runtimeAttrsConfig), + validatedRuntimeAttributes + ) val noAddress: Boolean = RuntimeAttributesValidation.extract(noAddressValidation(runtimeAttrsConfig), validatedRuntimeAttributes) val useDockerImageCache: Option[Boolean] = RuntimeAttributesValidation.extractOption( @@ -242,6 +250,7 @@ object PipelinesApiRuntimeAttributes { docker, failOnStderr, continueOnReturnCode, + returnCodes, noAddress, googleLegacyMachineSelection, useDockerImageCache, diff --git a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala index a7a880cb28c..d99617541bd 100644 --- a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala +++ b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala @@ -4,7 +4,7 @@ import cats.data.NonEmptyList import cromwell.backend.RuntimeAttributeDefinition import cromwell.backend.google.pipelines.common.PipelinesApiTestConfig.{googleConfiguration, papiAttributes, _} import cromwell.backend.google.pipelines.common.io.{DiskType, PipelinesApiAttachedDisk, PipelinesApiWorkingDisk} -import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet} +import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet, ReturnCodesSet, ReturnCodesString} import cromwell.core.WorkflowOptions import eu.timepit.refined.refineMV import org.scalatest.TestSuite @@ -103,6 +103,44 @@ final class PipelinesApiRuntimeAttributesSpec ) } + "validate a valid returnCodes integer entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomInteger(1)) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1))) + assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + } + + "validate a valid returnCodes String entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("*")) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesString("*")) + assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + } + + "validate a valid returnCodes array entry" in { + val runtimeAttributes = + Map("docker" -> WomString("ubuntu:latest"), + "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) + ) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1, 2))) + assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + } + + "coerce then validate a valid returnCodes array entry" in { + val runtimeAttributes = + Map("docker" -> WomString("ubuntu:latest"), + "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) + ) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1, 2))) + assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + } + + "fail to validate an invalid returnCodes entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("value")) + assertPapiRuntimeAttributesFailedCreation( + runtimeAttributes, + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + ) + } + "validate a valid cpu entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "cpu" -> WomInteger(2)) val expectedRuntimeAttributes = expectedDefaults.copy(cpu = refineMV(2)) @@ -327,6 +365,7 @@ trait PipelinesApiRuntimeAttributesSpecsMixin { this: TestSuite => "ubuntu:latest", false, ContinueOnReturnCodeSet(Set(0)), + ReturnCodesSet(Set(0)), false, false, None, diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala index 117e0ee79a7..6ebc9994922 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala @@ -23,6 +23,7 @@ import wom.values._ import java.util.regex.Pattern case class TesRuntimeAttributes(continueOnReturnCode: ContinueOnReturnCode, + returnCodes: ReturnCodes, dockerImage: String, dockerWorkingDir: Option[String], failOnStderr: Boolean, @@ -48,6 +49,9 @@ object TesRuntimeAttributes { private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig) + private def returnCodesValidation(runtimeConfig: Option[Config]) = + ReturnCodesValidation.default(runtimeConfig) + private def diskSizeValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[MemorySize] = MemoryValidation.optional(DiskSizeKey) @@ -158,6 +162,10 @@ object TesRuntimeAttributes { RuntimeAttributesValidation.extract(continueOnReturnCodeValidation(backendRuntimeConfig), validatedRuntimeAttributes ) + val returnCodes: ReturnCodes = + RuntimeAttributesValidation.extract(returnCodesValidation(backendRuntimeConfig), + validatedRuntimeAttributes + ) val preemptible: Boolean = RuntimeAttributesValidation.extract(preemptibleValidation(backendRuntimeConfig), validatedRuntimeAttributes) val localizedSas: Option[String] = @@ -175,6 +183,7 @@ object TesRuntimeAttributes { diskSizeCompatValidation(backendRuntimeConfig), failOnStderrValidation(backendRuntimeConfig), continueOnReturnCodeValidation(backendRuntimeConfig), + returnCodesValidation(backendRuntimeConfig), preemptibleValidation(backendRuntimeConfig), localizedSasValidation ) @@ -186,6 +195,7 @@ object TesRuntimeAttributes { new TesRuntimeAttributes( continueOnReturnCode, + returnCodes, docker, dockerWorkingDir, failOnStderr, diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala index 1fb43503fb6..b62f26f2c21 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala @@ -1,7 +1,7 @@ package cromwell.backend.impl.tes import common.assertion.CromwellTimeoutSpec -import cromwell.backend.validation.ContinueOnReturnCodeSet +import cromwell.backend.validation.{ContinueOnReturnCodeSet, ReturnCodesSet, ReturnCodesString} import cromwell.backend.{BackendConfigurationDescriptor, RuntimeAttributeDefinition, TestConfig} import cromwell.core.WorkflowOptions import eu.timepit.refined.numeric.Positive @@ -18,6 +18,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec val expectedDefaults = new TesRuntimeAttributes( ContinueOnReturnCodeSet(Set(0)), + ReturnCodesSet(Set(0)), "ubuntu:latest", None, false, @@ -161,6 +162,48 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec ) } + "validate a valid returnCodes int entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomInteger(1)) + val expectedRuntimeAttributes = + expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodesSet(Set(1))) + assertSuccess(runtimeAttributes, expectedRuntimeAttributes) + } + + "validate a valid returnCodes String entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("*")) + val expectedRuntimeAttributes = + expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodesString("*")) + assertSuccess(runtimeAttributes, expectedRuntimeAttributes) + } + + "validate a valid returnCodes array entry" in { + val runtimeAttributes = + Map("docker" -> WomString("ubuntu:latest"), + "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) + ) + val expectedRuntimeAttributes = + expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodesSet(Set(1, 2))) + assertSuccess(runtimeAttributes, expectedRuntimeAttributes) + } + + "coerce then validate a valid returnCodes array entry" in { + val runtimeAttributes = + Map("docker" -> WomString("ubuntu:latest"), + "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) + ) + val expectedRuntimeAttributes = + expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodesSet(Set(1, 2))) + assertSuccess(runtimeAttributes, expectedRuntimeAttributes) + } + + "fail to validate an invalid returnCodes entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("value")) + assertFailure( + runtimeAttributes, + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + ) + } + "validate a valid cpu entry" in assertSuccess( Map("docker" -> WomString("ubuntu:latest"), "cpu" -> WomInteger(2)), expectedDefaultsPlusUbuntuDocker.copy(cpu = Option(refineMV[Positive](2))) diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala index 27e7a058d5d..6defd55c712 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala @@ -2,7 +2,7 @@ package cromwell.backend.impl.tes import common.assertion.CromwellTimeoutSpec import common.mock.MockSugar -import cromwell.backend.validation.ContinueOnReturnCodeSet +import cromwell.backend.validation.{ContinueOnReturnCodeSet, ReturnCodesSet} import cromwell.backend.{BackendSpec, BackendWorkflowDescriptor, TestConfig} import cromwell.core.{RootWorkflowId, WorkflowId, WorkflowOptions} import cromwell.core.labels.Labels @@ -19,6 +19,7 @@ class TesTaskSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit val runtimeAttributes = new TesRuntimeAttributes( ContinueOnReturnCodeSet(Set(0)), + ReturnCodesSet(Set(0)), "ubuntu:latest", None, false, diff --git a/wom/src/main/scala/wom/RuntimeAttributes.scala b/wom/src/main/scala/wom/RuntimeAttributes.scala index 2c66b6859d3..4de582f5e1b 100644 --- a/wom/src/main/scala/wom/RuntimeAttributes.scala +++ b/wom/src/main/scala/wom/RuntimeAttributes.scala @@ -15,6 +15,9 @@ object RuntimeAttributesKeys { val MemoryKey = "memory" val FailOnStderrKey = "failOnStderr" val ContinueOnReturnCodeKey = "continueOnReturnCode" + + // New for WDL 1.1 + val ReturnCodesKey = "returnCodes" } case class RuntimeAttributes(attributes: Map[String, WomExpression]) From a183b053b31300eb5b6daaf97854c7f07bfb4ba0 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Mon, 18 Mar 2024 12:32:18 -0400 Subject: [PATCH 02/23] scalafmt --- .../standard/StandardAsyncExecutionActor.scala | 5 ++--- ...tandardValidatedRuntimeAttributesBuilderSpec.scala | 4 ++-- .../common/PipelinesApiRuntimeAttributesSpec.scala | 11 ++++++++--- .../backend/impl/tes/TesRuntimeAttributes.scala | 4 +--- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala index 27085121c91..a6a79db9f8d 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala @@ -1431,9 +1431,8 @@ trait StandardAsyncExecutionActor } else { tryReturnCodeAsInt match { case Success(returnCodeAsInt) - if outOfMemoryDetected && memoryRetryRequested && (!continueOnReturnCode.continueFor( - returnCodeAsInt) - || !returnCode.continueFor(returnCodeAsInt)) => + if outOfMemoryDetected && memoryRetryRequested && (!continueOnReturnCode.continueFor(returnCodeAsInt) + || !returnCode.continueFor(returnCodeAsInt)) => val executionHandle = Future.successful( FailedNonRetryableExecutionHandle( RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log), diff --git a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala index 5c3573c5b0c..735fe4e115e 100644 --- a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala @@ -176,8 +176,8 @@ class StandardValidatedRuntimeAttributesBuilderSpec ) val runtimeAttributes = Map.empty[String, WomValue] assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, - expectedRuntimeAttributes, - workflowOptions = workflowOptions + expectedRuntimeAttributes, + workflowOptions = workflowOptions ) } diff --git a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala index d99617541bd..2634ebf96d6 100644 --- a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala +++ b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala @@ -4,7 +4,12 @@ import cats.data.NonEmptyList import cromwell.backend.RuntimeAttributeDefinition import cromwell.backend.google.pipelines.common.PipelinesApiTestConfig.{googleConfiguration, papiAttributes, _} import cromwell.backend.google.pipelines.common.io.{DiskType, PipelinesApiAttachedDisk, PipelinesApiWorkingDisk} -import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet, ReturnCodesSet, ReturnCodesString} +import cromwell.backend.validation.{ + ContinueOnReturnCodeFlag, + ContinueOnReturnCodeSet, + ReturnCodesSet, + ReturnCodesString +} import cromwell.core.WorkflowOptions import eu.timepit.refined.refineMV import org.scalatest.TestSuite @@ -118,7 +123,7 @@ final class PipelinesApiRuntimeAttributesSpec "validate a valid returnCodes array entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), - "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) + "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1, 2))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) @@ -127,7 +132,7 @@ final class PipelinesApiRuntimeAttributesSpec "coerce then validate a valid returnCodes array entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), - "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) + "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1, 2))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala index 6ebc9994922..913e2fb1734 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala @@ -163,9 +163,7 @@ object TesRuntimeAttributes { validatedRuntimeAttributes ) val returnCodes: ReturnCodes = - RuntimeAttributesValidation.extract(returnCodesValidation(backendRuntimeConfig), - validatedRuntimeAttributes - ) + RuntimeAttributesValidation.extract(returnCodesValidation(backendRuntimeConfig), validatedRuntimeAttributes) val preemptible: Boolean = RuntimeAttributesValidation.extract(preemptibleValidation(backendRuntimeConfig), validatedRuntimeAttributes) val localizedSas: Option[String] = From d106db059fc64adceb3c342c4ea967f6d9fbbf21 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Mon, 18 Mar 2024 12:44:15 -0400 Subject: [PATCH 03/23] Renamed returnCodes directory to return_codes --- .../{returnCodes => return_codes}/return_codes.wdl | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename centaur/src/main/resources/standardTestCases/{returnCodes => return_codes}/return_codes.wdl (100%) diff --git a/centaur/src/main/resources/standardTestCases/returnCodes/return_codes.wdl b/centaur/src/main/resources/standardTestCases/return_codes/return_codes.wdl similarity index 100% rename from centaur/src/main/resources/standardTestCases/returnCodes/return_codes.wdl rename to centaur/src/main/resources/standardTestCases/return_codes/return_codes.wdl From f3f99725cb80ff1ddd7f20add1db44d99248a0ce Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Mon, 18 Mar 2024 13:15:07 -0400 Subject: [PATCH 04/23] Fixed failing unit test --- .../batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala index f8d6e071b31..9cfea1dd925 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala @@ -939,6 +939,7 @@ class GcpBatchAsyncBackendJobExecutionActorSpec "runtimeAttributes:disks" -> "local-disk 200 SSD", "runtimeAttributes:docker" -> "ubuntu:latest", "runtimeAttributes:failOnStderr" -> "false", + "runtimeAttributes:returnCodes" -> "0", "runtimeAttributes:memory" -> "2 GB", "runtimeAttributes:noAddress" -> "false", "runtimeAttributes:preemptible" -> "0", From 4f1cadbbdbb70c52b0126c20add5d9facc18c978 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Mon, 18 Mar 2024 13:43:02 -0400 Subject: [PATCH 05/23] Fixed failing unit test --- .../common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala index 41e826a4dc2..992ac594f97 100644 --- a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala @@ -1726,6 +1726,7 @@ class PipelinesApiAsyncBackendJobExecutionActorSpec "runtimeAttributes:disks" -> "local-disk 200 SSD", "runtimeAttributes:docker" -> "ubuntu:latest", "runtimeAttributes:failOnStderr" -> "false", + "runtimeAttributes:returnCodes" -> "0", "runtimeAttributes:memory" -> "2 GB", "runtimeAttributes:noAddress" -> "false", "runtimeAttributes:preemptible" -> "0", From d830f4809aee2fe00b8b4526cef9fed1e18704be Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Thu, 21 Mar 2024 09:47:41 -0400 Subject: [PATCH 06/23] Abstracted duplicated code for returnCodes and continueOnReturnCode and more testing --- .../StandardAsyncExecutionActor.scala | 19 +++++-- .../validation/ContinueOnReturnCode.scala | 17 +----- .../ContinueOnReturnCodeValidation.scala | 27 ++------- .../backend/validation/ReturnCode.scala | 34 ++++++++++++ .../validation/ReturnCodeValidation.scala | 39 +++++++++++++ .../backend/validation/ReturnCodes.scala | 17 +----- .../validation/ReturnCodesValidation.scala | 40 ++++---------- .../RuntimeAttributesValidation.scala | 4 +- ...ckendWorkflowInitializationActorSpec.scala | 19 +++---- ...alidatedRuntimeAttributesBuilderSpec.scala | 12 ++-- .../validation/ContinueOnReturnCodeSpec.scala | 2 +- .../backend/validation/ReturnCodesSpec.scala | 39 +++++++++++++ .../RuntimeAttributesValidationSpec.scala | 8 +-- ...urn_codes_and_continue_on_return_code.test | 11 ++++ ...turn_codes_and_continue_on_return_code.wdl | 21 +++++++ ...turn_codes_and_continue_on_return_code.wdl | 55 +++++++++++++++++++ ...rn_codes_and_continue_on_return_codes.test | 11 ++++ .../impl/aws/AwsBatchRuntimeAttributes.scala | 8 +-- .../backend/impl/aws/AwsBatchJobSpec.scala | 4 +- .../aws/AwsBatchRuntimeAttributesSpec.scala | 27 ++++----- .../models/GcpBatchRuntimeAttributes.scala | 8 +-- .../GcpBatchRuntimeAttributesSpec.scala | 6 +- .../PipelinesApiRuntimeAttributes.scala | 8 +-- .../PipelinesApiRuntimeAttributesSpec.scala | 23 +++----- .../impl/tes/TesRuntimeAttributes.scala | 8 +-- .../impl/tes/TesRuntimeAttributesSpec.scala | 18 +++--- .../backend/impl/tes/TesTaskSpec.scala | 6 +- 27 files changed, 320 insertions(+), 171 deletions(-) create mode 100644 backend/src/main/scala/cromwell/backend/validation/ReturnCode.scala create mode 100644 backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala create mode 100644 backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala create mode 100644 centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code.test create mode 100644 centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl create mode 100644 centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl create mode 100644 centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_codes.test diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala index a6a79db9f8d..ecc0a34fa0c 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala @@ -752,7 +752,7 @@ trait StandardAsyncExecutionActor * * @return the behavior for continuing on the return code. */ - lazy val continueOnReturnCode: ContinueOnReturnCode = + lazy val continueOnReturnCode: ReturnCode = RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) /** @@ -760,9 +760,18 @@ trait StandardAsyncExecutionActor * * @return the behavior for continuing on the return code. */ - lazy val returnCode: ReturnCodes = + lazy val returnCodes: ReturnCode = RuntimeAttributesValidation.extract(ReturnCodesValidation.instance, validatedRuntimeAttributes) + /** + * Returns the true behavior for continuing on the return code. If `returnCodes` runtime attribute is specified, + * then `continueOnReturnCode` attribute is ignored. If `returnCodes` is unspecified, then the value in + * `continueOnReturnCode` will be used. + */ + lazy val returnCode: ReturnCode = + if (returnCodes.isInstanceOf[ReturnCodeSet] && returnCodes.equals(ReturnCodeSet(Set(0)))) continueOnReturnCode + else returnCodes + /** * Returns the max number of times that a failed job should be retried, obtained by converting `maxRetries` to an Int. */ @@ -1396,8 +1405,7 @@ trait StandardAsyncExecutionActor ) ) retryElseFail(executionHandle) - case Success(returnCodeAsInt) - if continueOnReturnCode.continueFor(returnCodeAsInt) || returnCode.continueFor(returnCodeAsInt) => + case Success(returnCodeAsInt) if returnCode.continueFor(returnCodeAsInt) => handleExecutionSuccess(status, oldHandle, returnCodeAsInt) // It's important that we check retryWithMoreMemory case before isAbort. RC could be 137 in either case; // if it was caused by OOM killer, want to handle as OOM and not job abort. @@ -1431,8 +1439,7 @@ trait StandardAsyncExecutionActor } else { tryReturnCodeAsInt match { case Success(returnCodeAsInt) - if outOfMemoryDetected && memoryRetryRequested && (!continueOnReturnCode.continueFor(returnCodeAsInt) - || !returnCode.continueFor(returnCodeAsInt)) => + if outOfMemoryDetected && memoryRetryRequested && !returnCode.continueFor(returnCodeAsInt) => val executionHandle = Future.successful( FailedNonRetryableExecutionHandle( RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log), diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala index c314a660122..1c850d71adb 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala @@ -9,7 +9,7 @@ object ContinueOnReturnCode { /** * Decides if a call/job continues upon a specific return code. */ -sealed trait ContinueOnReturnCode { +sealed trait ContinueOnReturnCode extends ReturnCode { /** * Returns true if the call is a success based on the return code. @@ -17,10 +17,10 @@ sealed trait ContinueOnReturnCode { * @param returnCode Return code from the process / script. * @return True if the call is a success. */ - final def continueFor(returnCode: Int): Boolean = + final override def continueFor(returnCode: Int): Boolean = this match { case ContinueOnReturnCodeFlag(continue) => continue || returnCode == 0 - case ContinueOnReturnCodeSet(returnCodes) => returnCodes.contains(returnCode) + case _ => super.continueFor(returnCode) } } @@ -31,14 +31,3 @@ sealed trait ContinueOnReturnCode { case class ContinueOnReturnCodeFlag(continue: Boolean) extends ContinueOnReturnCode { override def toString = continue.toString } - -/** - * Continues only if the call/job return code is found in returnCodes. - * @param returnCodes Inclusive set of return codes that specify a job success. - */ -case class ContinueOnReturnCodeSet(returnCodes: Set[Int]) extends ContinueOnReturnCode { - override def toString = returnCodes match { - case single if single.size == 1 => returnCodes.head.toString - case multiple => s"[${multiple.mkString(",")}]" - } -} diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala index 14dfb369d77..100cb965806 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala @@ -1,9 +1,7 @@ package cromwell.backend.validation -import cats.data.Validated.{Invalid, Valid} import cats.implicits._ import com.typesafe.config.Config -import cromwell.backend.validation.RuntimeAttributesValidation._ import common.validation.ErrorOr._ import wom.RuntimeAttributesKeys import wom.types._ @@ -23,42 +21,29 @@ import scala.util.Try * `default` a validation with the default value specified by the reference.conf file. */ object ContinueOnReturnCodeValidation { - lazy val instance: RuntimeAttributesValidation[ContinueOnReturnCode] = new ContinueOnReturnCodeValidation - def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ContinueOnReturnCode] = + lazy val instance: RuntimeAttributesValidation[ReturnCode] = new ContinueOnReturnCodeValidation + def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ReturnCode] = instance.withDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(runtimeConfig) } -class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[ContinueOnReturnCode] { +class ContinueOnReturnCodeValidation extends ReturnCodeValidation { override def key: String = RuntimeAttributesKeys.ContinueOnReturnCodeKey override def coercion: Set[WomType] = ContinueOnReturnCode.validWdlTypes - override def validateValue: PartialFunction[WomValue, ErrorOr[ContinueOnReturnCode]] = { + override def validateValue: PartialFunction[WomValue, ErrorOr[ReturnCode]] = { case WomBoolean(value) => ContinueOnReturnCodeFlag(value).validNel case WomString(value) if Try(value.toBoolean).isSuccess => ContinueOnReturnCodeFlag(value.toBoolean).validNel - case WomString(value) if Try(value.toInt).isSuccess => ContinueOnReturnCodeSet(Set(value.toInt)).validNel - case WomInteger(value) => ContinueOnReturnCodeSet(Set(value)).validNel - case value @ WomArray(_, seq) => - val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int] - errorOrInts match { - case Valid(ints) => ContinueOnReturnCodeSet(ints.toSet).validNel - case Invalid(_) => invalidValueFailure(value) - } + case value => super.validateValue(value) } override def validateExpression: PartialFunction[WomValue, Boolean] = { case WomBoolean(_) => true - case WomString(value) if Try(value.toInt).isSuccess => true case WomString(value) if Try(value.toBoolean).isSuccess => true - case WomInteger(_) => true - case WomArray(WomArrayType(WomStringType), elements) => - elements forall { value => - Try(value.valueString.toInt).isSuccess - } - case WomArray(WomArrayType(WomIntegerType), _) => true + case value => super.validateExpression(value) } override protected def missingValueMessage: String = s"Expecting $key" + diff --git a/backend/src/main/scala/cromwell/backend/validation/ReturnCode.scala b/backend/src/main/scala/cromwell/backend/validation/ReturnCode.scala new file mode 100644 index 00000000000..a1a29e05f11 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/validation/ReturnCode.scala @@ -0,0 +1,34 @@ +package cromwell.backend.validation + +trait ReturnCode { + + /** + * Returns true if the call is a success based on the return code. + * + * @param returnCode Return code from the process / script. + * @return True if the call is a success. + */ + def continueFor(returnCode: Int): Boolean = + this match { + case ReturnCodeSet(returnCodes) => returnCodes.contains(returnCode) + } +} + +/** + * Continues only if the call/job return code is found in returnCodes. + * @param returnCodes Inclusive set of return codes that specify a job success. + */ +case class ReturnCodeSet(returnCodes: Set[Int]) extends ReturnCode { + override def toString = returnCodes match { + case single if single.size == 1 => returnCodes.head.toString + case multiple => s"[${multiple.mkString(",")}]" + } + + /** + * Returns true if the call is a success based on the return code. + * + * @param returnCode Return code from the process / script. + * @return True if the call is a success. + */ + override def continueFor(returnCode: Int): Boolean = super.continueFor(returnCode) +} diff --git a/backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala new file mode 100644 index 00000000000..f2014495b81 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala @@ -0,0 +1,39 @@ +package cromwell.backend.validation + +import cats.data.Validated.{Invalid, Valid} +import cats.implicits.{catsSyntaxValidatedId, toTraverseOps} +import common.validation.ErrorOr.ErrorOr +import cromwell.backend.validation.RuntimeAttributesValidation.validateInt +import wom.types.{WomArrayType, WomIntegerType, WomStringType, WomType} +import wom.values.{WomArray, WomInteger, WomString, WomValue} + +import scala.util.Try + +trait ReturnCodeValidation extends RuntimeAttributesValidation[ReturnCode] { + override def key: String + override def coercion: Iterable[WomType] + override def validateValue: PartialFunction[WomValue, ErrorOr[ReturnCode]] = { + case WomString(value) if Try(value.toInt).isSuccess => ReturnCodeSet(Set(value.toInt)).validNel + case WomInteger(value) => ReturnCodeSet(Set(value)).validNel + case value @ WomArray(_, seq) => + val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int] + errorOrInts match { + case Valid(ints) => ReturnCodeSet(ints.toSet).validNel + case Invalid(_) => invalidValueFailure(value) + } + case value => invalidValueFailure(value) + } + + override def validateExpression: PartialFunction[WomValue, Boolean] = { + case WomString(value) if Try(value.toInt).isSuccess => true + case WomInteger(_) => true + case WomArray(WomArrayType(WomStringType), elements) => + elements forall { value => + Try(value.valueString.toInt).isSuccess + } + case WomArray(WomArrayType(WomIntegerType), _) => true + case _ => false + } + + override def usedInCallCaching: Boolean = true +} diff --git a/backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala b/backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala index d97a5aff6c8..5cef8346ad1 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala @@ -9,7 +9,7 @@ object ReturnCodes { /** * Decides if a call/job continues upon a specific return code. */ -sealed trait ReturnCodes { +sealed trait ReturnCodes extends ReturnCode { /** * Returns true if the call is a success based on the return code. @@ -17,10 +17,10 @@ sealed trait ReturnCodes { * @param returnCode Return code from the process / script. * @return True if the call is a success. */ - final def continueFor(returnCode: Int): Boolean = + final override def continueFor(returnCode: Int): Boolean = this match { case ReturnCodesString(continue) => continue.equals("*") || returnCode == 0 - case ReturnCodesSet(returnCodes) => returnCodes.contains(returnCode) + case _ => super.continueFor(returnCode) } } @@ -31,14 +31,3 @@ sealed trait ReturnCodes { case class ReturnCodesString(returnCode: String) extends ReturnCodes { override def toString = returnCode } - -/** - * Continues only if the call/job return code is found in returnCodes. - * @param returnCodes Inclusive set of return codes that specify a job success. - */ -case class ReturnCodesSet(returnCodes: Set[Int]) extends ReturnCodes { - override def toString = returnCodes match { - case single if single.size == 1 => returnCodes.head.toString - case multiple => s"[${multiple.mkString(",")}]" - } -} diff --git a/backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala index 755e9a658fa..c3e78443eb4 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala @@ -1,16 +1,11 @@ package cromwell.backend.validation -import cats.data.Validated.{Invalid, Valid} -import cats.implicits.{catsSyntaxValidatedId, toTraverseOps} +import cats.implicits.catsSyntaxValidatedId import com.typesafe.config.Config import common.validation.ErrorOr.ErrorOr -import cromwell.backend.validation.RuntimeAttributesValidation.validateInt import wom.RuntimeAttributesKeys -import wom.RuntimeAttributesKeys._ -import wom.types.{WomArrayType, WomIntegerType, WomStringType, WomType} -import wom.values.{WomArray, WomInteger, WomString, WomValue} - -import scala.util.Try +import wom.types.WomType +import wom.values.{WomInteger, WomString, WomValue} /** * Validates the "returnCodes" runtime attribute as an Integer, returning the value as an `Int`, or as @@ -22,9 +17,9 @@ import scala.util.Try * reference.conf file, coerced into a WomValue. */ object ReturnCodesValidation { - lazy val instance: RuntimeAttributesValidation[ReturnCodes] = new ReturnCodesValidation(ReturnCodesKey) - lazy val optional: OptionalRuntimeAttributesValidation[ReturnCodes] = instance.optional - def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ReturnCodes] = + lazy val instance: RuntimeAttributesValidation[ReturnCode] = new ReturnCodesValidation + lazy val optional: OptionalRuntimeAttributesValidation[ReturnCode] = instance.optional + def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ReturnCode] = instance.withDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = @@ -32,36 +27,21 @@ object ReturnCodesValidation { def configDefaultWomValue(config: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(config) } -class ReturnCodesValidation(attributeName: String) extends RuntimeAttributesValidation[ReturnCodes] { +class ReturnCodesValidation extends ReturnCodeValidation { override def key: String = RuntimeAttributesKeys.ReturnCodesKey override def coercion: Set[WomType] = ReturnCodes.validWdlTypes - override def validateValue: PartialFunction[WomValue, ErrorOr[ReturnCodes]] = { + override def validateValue: PartialFunction[WomValue, ErrorOr[ReturnCode]] = { case WomString(value) if value.equals("*") => ReturnCodesString(value).validNel - case WomString(value) if Try(value.toInt).isSuccess => - ReturnCodesSet(Set(value.toInt)).validNel - case WomInteger(value) => - ReturnCodesSet(Set(value)).validNel - case value @ WomArray(_, seq) => - val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int] - errorOrInts match { - case Valid(ints) => ReturnCodesSet(ints.toSet).validNel - case Invalid(_) => invalidValueFailure(value) - } + case value => super.validateValue(value) } override def validateExpression: PartialFunction[WomValue, Boolean] = { - case WomString(value) if Try(value.toInt).isSuccess => true - case WomInteger(_) => true case WomString(value) if value.equals("*") => true - case WomArray(WomArrayType(WomStringType), elements) => - elements forall { value => - Try(value.valueString.toInt).isSuccess - } - case WomArray(WomArrayType(WomIntegerType), _) => true + case value => super.validateExpression(value) } override protected def missingValueMessage: String = s"Expecting $key" + diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala index 54b864f4471..7cb513aad72 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala @@ -34,10 +34,10 @@ object RuntimeAttributesValidation { def validateContinueOnReturnCode(value: Option[WomValue], onMissingKey: => ErrorOr[ContinueOnReturnCode] - ): ErrorOr[ContinueOnReturnCode] = + ): ErrorOr[ReturnCode] = validateWithValidation(value, ContinueOnReturnCodeValidation.instance, onMissingKey) - def validateReturnCodes(value: Option[WomValue], onMissingKey: => ErrorOr[ReturnCodes]): ErrorOr[ReturnCodes] = + def validateReturnCodes(value: Option[WomValue], onMissingKey: => ErrorOr[ReturnCode]): ErrorOr[ReturnCode] = validateWithValidation(value, ReturnCodesValidation.instance, onMissingKey) def validateMemory(value: Option[WomValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = diff --git a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala index e14b4737687..c7cde397b69 100644 --- a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala @@ -7,9 +7,8 @@ import akka.testkit.TestActorRef import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.validation.{ ContinueOnReturnCodeFlag, - ContinueOnReturnCodeSet, ContinueOnReturnCodeValidation, - ReturnCodesSet, + ReturnCodeSet, ReturnCodesString, ReturnCodesValidation } @@ -144,14 +143,14 @@ class BackendWorkflowInitializationActorSpec .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) - valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) + valid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) val returnCodesValid = ReturnCodesValidation .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) returnCodesValid.isValid should be(result) - returnCodesValid.toEither.toOption.get should be(ReturnCodesSet(Set(value))) + returnCodesValid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) } forAll(integerRows) { value => @@ -172,14 +171,14 @@ class BackendWorkflowInitializationActorSpec .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) - valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) + valid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) val returnCodesValid = ReturnCodesValidation .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) returnCodesValid.isValid should be(result) - returnCodesValid.toEither.toOption.get should be(ReturnCodesSet(Set(value))) + returnCodesValid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) } forAll(integerRows) { value => @@ -216,14 +215,14 @@ class BackendWorkflowInitializationActorSpec .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) - valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) + valid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) val returnCodesValid = ReturnCodesValidation .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) returnCodesValid.isValid should be(result) - returnCodesValid.toEither.toOption.get should be(ReturnCodesSet(Set(value))) + returnCodesValid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) } forAll(integerRows) { value => @@ -244,14 +243,14 @@ class BackendWorkflowInitializationActorSpec .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) - valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) + valid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) val returnCodesValid = ReturnCodesValidation .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) returnCodesValid.isValid should be(result) - returnCodesValid.toEither.toOption.get should be(ReturnCodesSet(Set(value))) + returnCodesValid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) } forAll(integerRows) { value => diff --git a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala index 735fe4e115e..818f29bbf51 100644 --- a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala @@ -40,7 +40,7 @@ class StandardValidatedRuntimeAttributesBuilderSpec """.stripMargin val defaultRuntimeAttributes: Map[String, Any] = - Map(DockerKey -> None, FailOnStderrKey -> false, ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(0))) + Map(DockerKey -> None, FailOnStderrKey -> false, ContinueOnReturnCodeKey -> ReturnCodeSet(Set(0))) def workflowOptionsWithDefaultRuntimeAttributes(defaults: Map[String, JsValue]): WorkflowOptions = WorkflowOptions(JsObject(Map("default_runtime_attributes" -> JsObject(defaults)))) @@ -128,7 +128,7 @@ class StandardValidatedRuntimeAttributesBuilderSpec "validate a valid continueOnReturnCode entry" in { val runtimeAttributes = Map("continueOnReturnCode" -> WomInteger(1)) val expectedRuntimeAttributes = - defaultRuntimeAttributes + (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1))) + defaultRuntimeAttributes + (ContinueOnReturnCodeKey -> ReturnCodeSet(Set(1))) assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -142,7 +142,7 @@ class StandardValidatedRuntimeAttributesBuilderSpec "use workflow options as default if continueOnReturnCode key is missing" in { val expectedRuntimeAttributes = defaultRuntimeAttributes + - (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1, 2))) + (ContinueOnReturnCodeKey -> ReturnCodeSet(Set(1, 2))) val workflowOptions = workflowOptionsWithDefaultRuntimeAttributes( Map(ContinueOnReturnCodeKey -> JsArray(Vector(JsNumber(1), JsNumber(2)))) ) @@ -156,7 +156,7 @@ class StandardValidatedRuntimeAttributesBuilderSpec "validate a valid returnCode entry" in { val runtimeAttributes = Map("returnCodes" -> WomInteger(1)) val expectedRuntimeAttributes = - defaultRuntimeAttributes + (ReturnCodesKey -> ReturnCodesSet(Set(1))) + defaultRuntimeAttributes + (ReturnCodesKey -> ReturnCodeSet(Set(1))) assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -170,7 +170,7 @@ class StandardValidatedRuntimeAttributesBuilderSpec "use workflow options as default if returnCode key is missing" in { val expectedRuntimeAttributes = defaultRuntimeAttributes + - (ReturnCodesKey -> ReturnCodesSet(Set(1, 2))) + (ReturnCodesKey -> ReturnCodeSet(Set(1, 2))) val workflowOptions = workflowOptionsWithDefaultRuntimeAttributes( Map(ReturnCodesKey -> JsArray(Vector(JsNumber(1), JsNumber(2)))) ) @@ -216,7 +216,7 @@ class StandardValidatedRuntimeAttributesBuilderSpec docker should be(expectedRuntimeAttributes(DockerKey).asInstanceOf[Option[String]]) failOnStderr should be(expectedRuntimeAttributes(FailOnStderrKey).asInstanceOf[Boolean]) continueOnReturnCode should be( - expectedRuntimeAttributes(ContinueOnReturnCodeKey).asInstanceOf[ContinueOnReturnCode] + expectedRuntimeAttributes(ContinueOnReturnCodeKey) ) () } diff --git a/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala b/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala index e0ff75d3176..1ceb1666d46 100644 --- a/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala @@ -34,7 +34,7 @@ class ContinueOnReturnCodeSpec extends AnyWordSpecLike with CromwellTimeoutSpec ) forAll(setTests) { (set, returnCode, expectedContinue) => - ContinueOnReturnCodeSet(set).continueFor(returnCode) should be(expectedContinue) + ReturnCodeSet(set).continueFor(returnCode) should be(expectedContinue) } } } diff --git a/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala b/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala new file mode 100644 index 00000000000..39ad7c6a5a5 --- /dev/null +++ b/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala @@ -0,0 +1,39 @@ +package cromwell.backend.validation + +import common.assertion.CromwellTimeoutSpec +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.prop.TableDrivenPropertyChecks._ +import org.scalatest.wordspec.AnyWordSpecLike + +class ReturnCodesSpec extends AnyWordSpecLike with CromwellTimeoutSpec with Matchers with BeforeAndAfterAll { + "Checking for return codes" should { + "continue on expected return code flags" in { + val flagTests = Table(("flag", "returnCode", "expectedContinue"), + ("*", 0, true), + ("*", 1, true), + ) + + forAll(flagTests) { (flag, returnCode, expectedContinue) => + ReturnCodesString(flag).continueFor(returnCode) should be(expectedContinue) + } + } + + "continue on expected return code sets" in { + val setTests = Table( + ("set", "returnCode", "expectedContinue"), + (Set(0), 0, true), + (Set(0), 1, false), + (Set(1), 0, false), + (Set(1), 1, true), + (Set(0, 1), 0, true), + (Set(0, 1), 1, true), + (Set(0, 1), 2, false) + ) + + forAll(setTests) { (set, returnCode, expectedContinue) => + ReturnCodeSet(set).continueFor(returnCode) should be(expectedContinue) + } + } + } +} diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala index 862aa001d99..284ae81efb9 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala @@ -169,7 +169,7 @@ class RuntimeAttributesValidationSpec "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel ) result match { - case Valid(x) => assert(x == ContinueOnReturnCodeSet(Set(12))) + case Valid(x) => assert(x == ReturnCodeSet(Set(12))) case Invalid(e) => fail(e.toList.mkString(" ")) } } @@ -196,7 +196,7 @@ class RuntimeAttributesValidationSpec "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel ) result match { - case Valid(x) => assert(x == ContinueOnReturnCodeSet(Set(1, 2))) + case Valid(x) => assert(x == ReturnCodeSet(Set(1, 2))) case Invalid(e) => fail(e.toList.mkString(" ")) } } @@ -247,7 +247,7 @@ class RuntimeAttributesValidationSpec "Failed to get returnCodes mandatory key from runtime attributes".invalidNel ) result match { - case Valid(x) => assert(x == ReturnCodesSet(Set(12))) + case Valid(x) => assert(x == ReturnCodeSet(Set(12))) case Invalid(e) => fail(e.toList.mkString(" ")) } } @@ -259,7 +259,7 @@ class RuntimeAttributesValidationSpec "Failed to get returnCode mandatory key from runtime attributes".invalidNel ) result match { - case Valid(x) => assert(x == ReturnCodesSet(Set(1, 2))) + case Valid(x) => assert(x == ReturnCodeSet(Set(1, 2))) case Invalid(e) => fail(e.toList.mkString(" ")) } } diff --git a/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code.test b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code.test new file mode 100644 index 00000000000..8f55af82262 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code.test @@ -0,0 +1,11 @@ +name: invalid_return_codes_and_continue_on_return_code +testFormat: workflowfailure + +files { + workflow: invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl +} + +metadata { + workflowName: InvalidReturnCodeAndContinueOnReturnCode + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl new file mode 100644 index 00000000000..3192880b1da --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl @@ -0,0 +1,21 @@ +version development-1.1 + +workflow InvalidReturnCodeAndContinueOnReturnCode { + call InvalidReturnCodeContinueOnReturnCode +} + +task InvalidReturnCodeContinueOnReturnCode { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [5, 10, 15] + continueOnReturnCodes: [1] + } +} \ No newline at end of file diff --git a/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl new file mode 100644 index 00000000000..2c9b4418298 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl @@ -0,0 +1,55 @@ +version development-1.1 + +workflow ValidReturnCodeAndContinueOnReturnCode { + call ReturnCodeContinueOnReturnCode1 + call ReturnCodeContinueOnReturnCode2 + call ReturnCodeContinueOnReturnCode3 +} + +task ReturnCodeContinueOnReturnCode1 { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1] + continueOnReturnCodes: [0] + } +} + +task ReturnCodeContinueOnReturnCode2 { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1] + continueOnReturnCodes: false + } +} + +task ReturnCodeContinueOnReturnCode3 { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1, 4, 7] + continueOnReturnCodes: [1, 3, 5] + } +} \ No newline at end of file diff --git a/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_codes.test b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_codes.test new file mode 100644 index 00000000000..1a8354b9482 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_codes.test @@ -0,0 +1,11 @@ +name: valid_return_codes_and_continue_on_return_code +testFormat: workflowsuccess + +files { + workflow: valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl +} + +metadata { + workflowName: ValidReturnCodeAndContinueOnReturnCode + status: Succeeded +} diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala index de1184f1afd..59c40cd6311 100755 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala @@ -69,8 +69,8 @@ case class AwsBatchRuntimeAttributes(cpu: Int Refined Positive, dockerImage: String, queueArn: String, failOnStderr: Boolean, - continueOnReturnCode: ContinueOnReturnCode, - returnCodes: ReturnCodes, + continueOnReturnCode: ReturnCode, + returnCodes: ReturnCode, noAddress: Boolean, scriptS3BucketName: String, fileSystem: String = "s3" @@ -189,11 +189,11 @@ object AwsBatchRuntimeAttributes { RuntimeAttributesValidation.extract(queueArnValidation(runtimeAttrsConfig), validatedRuntimeAttributes) val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(runtimeAttrsConfig), validatedRuntimeAttributes) - val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract( + val continueOnReturnCode: ReturnCode = RuntimeAttributesValidation.extract( continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) - val returnCodes: ReturnCodes = RuntimeAttributesValidation.extract( + val returnCodes: ReturnCode = RuntimeAttributesValidation.extract( returnCodesValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala index edf4f7dd9b7..d4aa5232e4c 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala @@ -34,7 +34,7 @@ package cromwell.backend.impl.aws import common.collections.EnhancedCollections._ import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.backend.BackendSpec._ -import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ReturnCodesSet} +import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ReturnCodeSet} import cromwell.core.TestKitSuite import cromwell.util.SampleWdl import eu.timepit.refined.api.Refined @@ -110,7 +110,7 @@ class AwsBatchJobSpec extends TestKitSuite with AnyFlatSpecLike with Matchers wi queueArn = "arn:aws:batch:us-east-1:123456789:job-queue/default-gwf-core", failOnStderr = true, continueOnReturnCode = ContinueOnReturnCodeFlag(false), - returnCodes = ReturnCodesSet(Set(0)), + returnCodes = ReturnCodeSet(Set(0)), noAddress = false, scriptS3BucketName = "script-bucket", fileSystem = "s3" diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala index 728d30f78f3..2837e3cd23c 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala @@ -35,12 +35,7 @@ import cats.data.NonEmptyList import common.assertion.CromwellTimeoutSpec import cromwell.backend.RuntimeAttributeDefinition import cromwell.backend.impl.aws.io.{AwsBatchVolume, AwsBatchWorkingDisk} -import cromwell.backend.validation.{ - ContinueOnReturnCodeFlag, - ContinueOnReturnCodeSet, - ReturnCodesSet, - ReturnCodesString -} +import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ReturnCodeSet, ReturnCodesString} import cromwell.core.WorkflowOptions import eu.timepit.refined.numeric.Positive import eu.timepit.refined.refineMV @@ -72,8 +67,8 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "ubuntu:latest", "arn:aws:batch:us-east-1:111222333444:job-queue/job-queue", false, - ContinueOnReturnCodeSet(Set(0)), - ReturnCodesSet(Set(0)), + ReturnCodeSet(Set(0)), + ReturnCodeSet(Set(0)), false, "my-stuff" ) @@ -86,8 +81,8 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "ubuntu:latest", "arn:aws:batch:us-east-1:111222333444:job-queue/job-queue", false, - ContinueOnReturnCodeSet(Set(0)), - ReturnCodesSet(Set(0)), + ReturnCodeSet(Set(0)), + ReturnCodeSet(Set(0)), false, "", "local" @@ -169,7 +164,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "continueOnReturnCode" -> WomInteger(1) ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ReturnCodeSet(Set(1))) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -188,7 +183,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "continueOnReturnCode" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ReturnCodeSet(Set(1, 2))) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -198,7 +193,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "continueOnReturnCode" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ReturnCodeSet(Set(1, 2))) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -218,7 +213,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "returnCodes" -> WomInteger(1) ) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1))) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodeSet(Set(1))) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -237,7 +232,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodeSet(Set(1, 2))) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -247,7 +242,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodeSet(Set(1, 2))) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala index 82a3371f207..fac2e8d557e 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala @@ -46,8 +46,8 @@ final case class GcpBatchRuntimeAttributes(cpu: Int Refined Positive, disks: Seq[GcpBatchAttachedDisk], dockerImage: String, failOnStderr: Boolean, - continueOnReturnCode: ContinueOnReturnCode, - returnCodes: ReturnCodes, + continueOnReturnCode: ReturnCode, + returnCodes: ReturnCode, noAddress: Boolean, useDockerImageCache: Option[Boolean], checkpointFilename: Option[String] @@ -211,11 +211,11 @@ object GcpBatchRuntimeAttributes { val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes) val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(runtimeAttrsConfig), validatedRuntimeAttributes) - val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract( + val continueOnReturnCode: ReturnCode = RuntimeAttributesValidation.extract( continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) - val returnCodes: ReturnCodes = RuntimeAttributesValidation.extract( + val returnCodes: ReturnCode = RuntimeAttributesValidation.extract( returnCodesValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala index 108dd3e8173..34024fe8f33 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala @@ -3,7 +3,7 @@ package cromwell.backend.google.batch.models import cats.data.NonEmptyList import cromwell.backend.RuntimeAttributeDefinition import cromwell.backend.google.batch.models.GcpBatchTestConfig._ -import cromwell.backend.validation.{ContinueOnReturnCodeSet, ReturnCodesSet} +import cromwell.backend.validation.ReturnCodeSet //import cromwell.backend.google.batch.io.{DiskType, GcpBatchAttachedDisk} import cromwell.backend.google.batch.io.{DiskType, GcpBatchWorkingDisk} import cromwell.core.WorkflowOptions @@ -282,8 +282,8 @@ trait GcpBatchRuntimeAttributesSpecsMixin { disks = Vector(GcpBatchWorkingDisk(DiskType.SSD, 10)), dockerImage = "ubuntu:latest", failOnStderr = false, - continueOnReturnCode = ContinueOnReturnCodeSet(Set(0)), - returnCodes = ReturnCodesSet(Set(0)), + continueOnReturnCode = ReturnCodeSet(Set(0)), + returnCodes = ReturnCodeSet(Set(0)), noAddress = false, useDockerImageCache = None, checkpointFilename = None diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala index 183dec9311a..a06d52c6b9e 100644 --- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala +++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala @@ -49,8 +49,8 @@ final case class PipelinesApiRuntimeAttributes(cpu: Int Refined Positive, disks: Seq[PipelinesApiAttachedDisk], dockerImage: String, failOnStderr: Boolean, - continueOnReturnCode: ContinueOnReturnCode, - returnCodes: ReturnCodes, + continueOnReturnCode: ReturnCode, + returnCodes: ReturnCode, noAddress: Boolean, googleLegacyMachineSelection: Boolean, useDockerImageCache: Option[Boolean], @@ -223,11 +223,11 @@ object PipelinesApiRuntimeAttributes { val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes) val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(runtimeAttrsConfig), validatedRuntimeAttributes) - val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract( + val continueOnReturnCode: ReturnCode = RuntimeAttributesValidation.extract( continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) - val returnCodes: ReturnCodes = RuntimeAttributesValidation.extract( + val returnCodes: ReturnCode = RuntimeAttributesValidation.extract( returnCodesValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) diff --git a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala index 2634ebf96d6..1267321cf6b 100644 --- a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala +++ b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala @@ -4,12 +4,7 @@ import cats.data.NonEmptyList import cromwell.backend.RuntimeAttributeDefinition import cromwell.backend.google.pipelines.common.PipelinesApiTestConfig.{googleConfiguration, papiAttributes, _} import cromwell.backend.google.pipelines.common.io.{DiskType, PipelinesApiAttachedDisk, PipelinesApiWorkingDisk} -import cromwell.backend.validation.{ - ContinueOnReturnCodeFlag, - ContinueOnReturnCodeSet, - ReturnCodesSet, - ReturnCodesString -} +import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ReturnCodeSet, ReturnCodesString} import cromwell.core.WorkflowOptions import eu.timepit.refined.refineMV import org.scalatest.TestSuite @@ -72,7 +67,7 @@ final class PipelinesApiRuntimeAttributesSpec "validate a valid continueOnReturnCode integer entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomInteger(1)) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ReturnCodeSet(Set(1))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -87,7 +82,7 @@ final class PipelinesApiRuntimeAttributesSpec Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ReturnCodeSet(Set(1, 2))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -96,7 +91,7 @@ final class PipelinesApiRuntimeAttributesSpec Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ReturnCodeSet(Set(1, 2))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -110,7 +105,7 @@ final class PipelinesApiRuntimeAttributesSpec "validate a valid returnCodes integer entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomInteger(1)) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1))) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodeSet(Set(1))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -125,7 +120,7 @@ final class PipelinesApiRuntimeAttributesSpec Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodeSet(Set(1, 2))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -134,7 +129,7 @@ final class PipelinesApiRuntimeAttributesSpec Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodeSet(Set(1, 2))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -369,8 +364,8 @@ trait PipelinesApiRuntimeAttributesSpecsMixin { this: TestSuite => Vector(PipelinesApiWorkingDisk(DiskType.SSD, 10)), "ubuntu:latest", false, - ContinueOnReturnCodeSet(Set(0)), - ReturnCodesSet(Set(0)), + ReturnCodeSet(Set(0)), + ReturnCodeSet(Set(0)), false, false, None, diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala index 913e2fb1734..409edd1adb9 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala @@ -22,8 +22,8 @@ import wom.values._ import java.util.regex.Pattern -case class TesRuntimeAttributes(continueOnReturnCode: ContinueOnReturnCode, - returnCodes: ReturnCodes, +case class TesRuntimeAttributes(continueOnReturnCode: ReturnCode, + returnCodes: ReturnCode, dockerImage: String, dockerWorkingDir: Option[String], failOnStderr: Boolean, @@ -158,11 +158,11 @@ object TesRuntimeAttributes { val disk: Option[MemorySize] = detectDiskFormat(backendRuntimeConfig, validatedRuntimeAttributes) val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(backendRuntimeConfig), validatedRuntimeAttributes) - val continueOnReturnCode: ContinueOnReturnCode = + val continueOnReturnCode: ReturnCode = RuntimeAttributesValidation.extract(continueOnReturnCodeValidation(backendRuntimeConfig), validatedRuntimeAttributes ) - val returnCodes: ReturnCodes = + val returnCodes: ReturnCode = RuntimeAttributesValidation.extract(returnCodesValidation(backendRuntimeConfig), validatedRuntimeAttributes) val preemptible: Boolean = RuntimeAttributesValidation.extract(preemptibleValidation(backendRuntimeConfig), validatedRuntimeAttributes) diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala index b62f26f2c21..f3d563d2672 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala @@ -1,7 +1,7 @@ package cromwell.backend.impl.tes import common.assertion.CromwellTimeoutSpec -import cromwell.backend.validation.{ContinueOnReturnCodeSet, ReturnCodesSet, ReturnCodesString} +import cromwell.backend.validation.{ReturnCodeSet, ReturnCodesString} import cromwell.backend.{BackendConfigurationDescriptor, RuntimeAttributeDefinition, TestConfig} import cromwell.core.WorkflowOptions import eu.timepit.refined.numeric.Positive @@ -17,8 +17,8 @@ import wom.values._ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec with Matchers { val expectedDefaults = new TesRuntimeAttributes( - ContinueOnReturnCodeSet(Set(0)), - ReturnCodesSet(Set(0)), + ReturnCodeSet(Set(0)), + ReturnCodeSet(Set(0)), "ubuntu:latest", None, false, @@ -130,7 +130,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec "validate a valid continueOnReturnCode entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomInteger(1)) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) + expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ReturnCodeSet(Set(1))) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } @@ -140,7 +140,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec "continueOnReturnCode" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) + expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ReturnCodeSet(Set(1, 2))) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } @@ -150,7 +150,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec "continueOnReturnCode" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) + expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ReturnCodeSet(Set(1, 2))) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } @@ -165,7 +165,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec "validate a valid returnCodes int entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomInteger(1)) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodesSet(Set(1))) + expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodeSet(Set(1))) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } @@ -182,7 +182,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodesSet(Set(1, 2))) + expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodeSet(Set(1, 2))) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } @@ -192,7 +192,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodesSet(Set(1, 2))) + expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodeSet(Set(1, 2))) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala index 6defd55c712..ea6393f49a0 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala @@ -2,7 +2,7 @@ package cromwell.backend.impl.tes import common.assertion.CromwellTimeoutSpec import common.mock.MockSugar -import cromwell.backend.validation.{ContinueOnReturnCodeSet, ReturnCodesSet} +import cromwell.backend.validation.ReturnCodeSet import cromwell.backend.{BackendSpec, BackendWorkflowDescriptor, TestConfig} import cromwell.core.{RootWorkflowId, WorkflowId, WorkflowOptions} import cromwell.core.labels.Labels @@ -18,8 +18,8 @@ import java.util.UUID class TesTaskSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with BackendSpec with MockSugar { val runtimeAttributes = new TesRuntimeAttributes( - ContinueOnReturnCodeSet(Set(0)), - ReturnCodesSet(Set(0)), + ReturnCodeSet(Set(0)), + ReturnCodeSet(Set(0)), "ubuntu:latest", None, false, From 2c4d8b36fa8194570e90386a8aa8d520d95f24b0 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Thu, 21 Mar 2024 09:57:08 -0400 Subject: [PATCH 07/23] scalafmt --- .../scala/cromwell/backend/validation/ReturnCodesSpec.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala b/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala index 39ad7c6a5a5..7eb3694946a 100644 --- a/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala @@ -9,10 +9,7 @@ import org.scalatest.wordspec.AnyWordSpecLike class ReturnCodesSpec extends AnyWordSpecLike with CromwellTimeoutSpec with Matchers with BeforeAndAfterAll { "Checking for return codes" should { "continue on expected return code flags" in { - val flagTests = Table(("flag", "returnCode", "expectedContinue"), - ("*", 0, true), - ("*", 1, true), - ) + val flagTests = Table(("flag", "returnCode", "expectedContinue"), ("*", 0, true), ("*", 1, true)) forAll(flagTests) { (flag, returnCode, expectedContinue) => ReturnCodesString(flag).continueFor(returnCode) should be(expectedContinue) From 8e1f888f467378f92172ad323ab4ae24e04da447 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Thu, 21 Mar 2024 10:52:57 -0400 Subject: [PATCH 08/23] Added comment --- .../cromwell/backend/validation/ReturnCodeValidation.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala index f2014495b81..14605cca41d 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala @@ -9,6 +9,10 @@ import wom.values.{WomArray, WomInteger, WomString, WomValue} import scala.util.Try +/** + * Validates the `continueOnReturnCode` and `returnCodes` runtime attributes, since they share the functionality of + * being a Set of Integers. + */ trait ReturnCodeValidation extends RuntimeAttributesValidation[ReturnCode] { override def key: String override def coercion: Iterable[WomType] From f2b548c7869242f95aa20f281c93217e7d264800 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Fri, 22 Mar 2024 13:12:54 -0400 Subject: [PATCH 09/23] Added step to SSH into VM running integration tests --- .github/workflows/integration_tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 61ed3c5af41..6569b1c72ad 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -104,6 +104,14 @@ jobs: set -e echo Running test.sh ./src/ci/bin/test.sh + # If a build step fails, activate SSH and idle for 30 minutes + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 30 + with: + limit-access-to-actor: true + detached: true # always() is some github magic that forces the following step to run, even when the previous fails. # Without it, the if statement won't be evaluated on a test failure. - uses: ravsamhq/notify-slack-action@v2 From b731aec3197724fa7d382b2fdd69851431ca942e Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Fri, 22 Mar 2024 13:40:56 -0400 Subject: [PATCH 10/23] Comment change to re-run tests --- .github/workflows/integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 6569b1c72ad..11f177a46f1 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -104,11 +104,11 @@ jobs: set -e echo Running test.sh ./src/ci/bin/test.sh - # If a build step fails, activate SSH and idle for 30 minutes + # If a build step fails, activate SSH and idle for 30 minutes. - name: Setup tmate session if: ${{ failure() }} uses: mxschmitt/action-tmate@v3 - timeout-minutes: 30 + timeout-minutes: 90 with: limit-access-to-actor: true detached: true From 23c6c07bd9dd18e95592395fd006a399147c9019 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Fri, 29 Mar 2024 16:21:21 -0400 Subject: [PATCH 11/23] Refactored to reduce code duplication --- .github/workflows/integration_tests.yml | 16 +- .../BackendWorkflowInitializationActor.scala | 12 +- .../StandardAsyncExecutionActor.scala | 20 +-- ...ardValidatedRuntimeAttributesBuilder.scala | 1 - .../validation/ContinueOnReturnCode.scala | 17 +- .../ContinueOnReturnCodeValidation.scala | 45 ++++-- .../backend/validation/ReturnCode.scala | 34 ---- .../validation/ReturnCodeValidation.scala | 43 ------ .../backend/validation/ReturnCodes.scala | 33 ---- .../validation/ReturnCodesValidation.scala | 51 ------ .../RuntimeAttributesValidation.scala | 8 +- .../TwoKeyRuntimeAttributesValidation.scala | 145 ++++++++++++++++++ .../ValidatedRuntimeAttributesBuilder.scala | 40 ++++- ...ckendWorkflowInitializationActorSpec.scala | 131 +++------------- ...alidatedRuntimeAttributesBuilderSpec.scala | 18 ++- .../validation/ContinueOnReturnCodeSpec.scala | 2 +- .../backend/validation/ReturnCodesSpec.scala | 19 +-- .../RuntimeAttributesValidationSpec.scala | 67 +------- .../impl/aws/AwsBatchRuntimeAttributes.scala | 14 +- .../backend/impl/aws/AwsBatchJobSpec.scala | 3 +- .../aws/AwsBatchRuntimeAttributesSpec.scala | 24 ++- .../models/GcpBatchRuntimeAttributes.scala | 13 +- .../GcpBatchRuntimeAttributesSpec.scala | 9 +- .../PipelinesApiRuntimeAttributes.scala | 13 +- .../PipelinesApiRuntimeAttributesSpec.scala | 23 ++- .../impl/tes/TesRuntimeAttributes.scala | 16 +- .../impl/tes/TesRuntimeAttributesSpec.scala | 23 ++- .../backend/impl/tes/TesTaskSpec.scala | 5 +- 28 files changed, 337 insertions(+), 508 deletions(-) delete mode 100644 backend/src/main/scala/cromwell/backend/validation/ReturnCode.scala delete mode 100644 backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala delete mode 100644 backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala delete mode 100644 backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala create mode 100644 backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 11f177a46f1..caf88d30dd1 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -96,6 +96,14 @@ jobs: - uses: ./.github/set_up_cromwell_action #This github action will set up git-secrets, caching, java, and sbt. with: cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + # If a build step fails, activate SSH and idle for 30 minutes. + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 90 + with: + limit-access-to-actor: true + detached: true #This script bascially just looks up another script to run, assuming that the other script's filename is: #src/ci/bin/test${BUILD_TYPE}.sh. The first letter of the BUILD_TYPE is automatically capitalized when looking. - name: Run Integration Test @@ -104,14 +112,6 @@ jobs: set -e echo Running test.sh ./src/ci/bin/test.sh - # If a build step fails, activate SSH and idle for 30 minutes. - - name: Setup tmate session - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - timeout-minutes: 90 - with: - limit-access-to-actor: true - detached: true # always() is some github magic that forces the following step to run, even when the previous fails. # Without it, the if statement won't be evaluated on a test failure. - uses: ravsamhq/notify-slack-action@v2 diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala index 8435a1019d2..9f59a451478 100644 --- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala @@ -11,7 +11,7 @@ import common.validation.Validation._ import cromwell.backend.BackendLifecycleActor._ import cromwell.backend.BackendWorkflowInitializationActor._ import cromwell.backend.async.{RuntimeAttributeValidationFailure, RuntimeAttributeValidationFailures} -import cromwell.backend.validation.{ContinueOnReturnCodeValidation, ReturnCodesValidation} +import cromwell.backend.validation.ContinueOnReturnCodeValidation import cromwell.core._ import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import cromwell.services.metadata.MetadataService.PutMetadataAction @@ -134,16 +134,6 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w .default(configurationDescriptor.backendRuntimeAttributesConfig) .validateOptionalWomValue(womExpressionMaybe) - /** - * This predicate is only appropriate for validation during workflow initialization. The logic does not differentiate - * between evaluation failures due to missing call inputs or evaluation failures due to malformed expressions, and will - * return `true` in both cases. - */ - protected def returnCodesPredicate(valueRequired: Boolean)(womExpressionMaybe: Option[WomValue]): Boolean = - ReturnCodesValidation - .default(configurationDescriptor.backendRuntimeAttributesConfig) - .validateOptionalWomValue(womExpressionMaybe) - protected def runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean] // FIXME: If a workflow executes jobs using multiple backends, diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala index ecc0a34fa0c..2e29ecf9273 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala @@ -37,6 +37,7 @@ import net.ceedubs.ficus.Ficus._ import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils import shapeless.Coproduct +import wom.RuntimeAttributesKeys.ReturnCodesKey import wom.callable.{AdHocValue, CommandTaskDefinition, ContainerizedInputExpression} import wom.expression.WomExpression import wom.graph.LocalName @@ -748,11 +749,11 @@ trait StandardAsyncExecutionActor RuntimeAttributesValidation.extract(FailOnStderrValidation.instance, validatedRuntimeAttributes) /** - * Returns the behavior for continuing on the return code, obtained by converting `returnCodeContents` to an Int. - * - * @return the behavior for continuing on the return code. - */ - lazy val continueOnReturnCode: ReturnCode = + * Returns the behavior for continuing on the return code, obtained by converting `returnCodeContents` to an Int. + * + * @return the behavior for continuing on the return code. + */ + lazy val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) /** @@ -760,16 +761,17 @@ trait StandardAsyncExecutionActor * * @return the behavior for continuing on the return code. */ - lazy val returnCodes: ReturnCode = - RuntimeAttributesValidation.extract(ReturnCodesValidation.instance, validatedRuntimeAttributes) + lazy val returnCodes: ContinueOnReturnCode = + RuntimeAttributesValidation.extract[ContinueOnReturnCode](ReturnCodesKey, validatedRuntimeAttributes) /** * Returns the true behavior for continuing on the return code. If `returnCodes` runtime attribute is specified, * then `continueOnReturnCode` attribute is ignored. If `returnCodes` is unspecified, then the value in * `continueOnReturnCode` will be used. */ - lazy val returnCode: ReturnCode = - if (returnCodes.isInstanceOf[ReturnCodeSet] && returnCodes.equals(ReturnCodeSet(Set(0)))) continueOnReturnCode + lazy val returnCode: ContinueOnReturnCode = + if (returnCodes.isInstanceOf[ContinueOnReturnCodeSet] && returnCodes.equals(ContinueOnReturnCodeSet(Set(0)))) + continueOnReturnCode else returnCodes /** diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala index cf8c672b6b8..77f890b4a19 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala @@ -32,7 +32,6 @@ object StandardValidatedRuntimeAttributesBuilder { def default(backendRuntimeConfig: Option[Config]): StandardValidatedRuntimeAttributesBuilder = { val required = Seq( ContinueOnReturnCodeValidation.default(backendRuntimeConfig), - ReturnCodesValidation.default(backendRuntimeConfig), FailOnStderrValidation.default(backendRuntimeConfig), MaxRetriesValidation.default(backendRuntimeConfig) ) diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala index 1c850d71adb..c314a660122 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala @@ -9,7 +9,7 @@ object ContinueOnReturnCode { /** * Decides if a call/job continues upon a specific return code. */ -sealed trait ContinueOnReturnCode extends ReturnCode { +sealed trait ContinueOnReturnCode { /** * Returns true if the call is a success based on the return code. @@ -17,10 +17,10 @@ sealed trait ContinueOnReturnCode extends ReturnCode { * @param returnCode Return code from the process / script. * @return True if the call is a success. */ - final override def continueFor(returnCode: Int): Boolean = + final def continueFor(returnCode: Int): Boolean = this match { case ContinueOnReturnCodeFlag(continue) => continue || returnCode == 0 - case _ => super.continueFor(returnCode) + case ContinueOnReturnCodeSet(returnCodes) => returnCodes.contains(returnCode) } } @@ -31,3 +31,14 @@ sealed trait ContinueOnReturnCode extends ReturnCode { case class ContinueOnReturnCodeFlag(continue: Boolean) extends ContinueOnReturnCode { override def toString = continue.toString } + +/** + * Continues only if the call/job return code is found in returnCodes. + * @param returnCodes Inclusive set of return codes that specify a job success. + */ +case class ContinueOnReturnCodeSet(returnCodes: Set[Int]) extends ContinueOnReturnCode { + override def toString = returnCodes match { + case single if single.size == 1 => returnCodes.head.toString + case multiple => s"[${multiple.mkString(",")}]" + } +} diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala index 100cb965806..e55a36a838b 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala @@ -1,8 +1,10 @@ package cromwell.backend.validation +import cats.data.Validated.{Invalid, Valid} import cats.implicits._ import com.typesafe.config.Config import common.validation.ErrorOr._ +import cromwell.backend.validation.RuntimeAttributesValidation.validateInt import wom.RuntimeAttributesKeys import wom.types._ import wom.values._ @@ -21,29 +23,54 @@ import scala.util.Try * `default` a validation with the default value specified by the reference.conf file. */ object ContinueOnReturnCodeValidation { - lazy val instance: RuntimeAttributesValidation[ReturnCode] = new ContinueOnReturnCodeValidation - def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ReturnCode] = - instance.withDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) + lazy val instance: TwoKeyRuntimeAttributesValidation[ContinueOnReturnCode, ContinueOnReturnCodeSet] = + new ContinueOnReturnCodeValidation + def default( + runtimeConfig: Option[Config] + ): TwoKeyRuntimeAttributesValidation[ContinueOnReturnCode, ContinueOnReturnCodeSet] = + instance.makeDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = - instance.configDefaultWomValue(runtimeConfig) + instance.configDefault(runtimeConfig) } -class ContinueOnReturnCodeValidation extends ReturnCodeValidation { +class ContinueOnReturnCodeValidation + extends TwoKeyRuntimeAttributesValidation[ContinueOnReturnCode, ContinueOnReturnCodeSet] { - override def key: String = RuntimeAttributesKeys.ContinueOnReturnCodeKey + override def key: String = RuntimeAttributesKeys.ReturnCodesKey + + override def altKey: String = RuntimeAttributesKeys.ContinueOnReturnCodeKey + + override def defaultVal: ContinueOnReturnCodeSet = ContinueOnReturnCodeSet(Set(0)) override def coercion: Set[WomType] = ContinueOnReturnCode.validWdlTypes - override def validateValue: PartialFunction[WomValue, ErrorOr[ReturnCode]] = { + override def validateValue: PartialFunction[WomValue, ErrorOr[ContinueOnReturnCode]] = { case WomBoolean(value) => ContinueOnReturnCodeFlag(value).validNel case WomString(value) if Try(value.toBoolean).isSuccess => ContinueOnReturnCodeFlag(value.toBoolean).validNel - case value => super.validateValue(value) + case WomString(value) if value.equals("*") => ContinueOnReturnCodeFlag(true).validNel + case WomString(value) if Try(value.toInt).isSuccess => ContinueOnReturnCodeSet(Set(value.toInt)).validNel + case WomInteger(value) => ContinueOnReturnCodeSet(Set(value)).validNel + case value @ WomArray(_, seq) => + val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int] + errorOrInts match { + case Valid(ints) => ContinueOnReturnCodeSet(ints.toSet).validNel + case Invalid(_) => invalidValueFailure(value) + } + case value => invalidValueFailure(value) } override def validateExpression: PartialFunction[WomValue, Boolean] = { case WomBoolean(_) => true case WomString(value) if Try(value.toBoolean).isSuccess => true - case value => super.validateExpression(value) + case WomString(value) if value.equals("*") => true + case WomString(value) if Try(value.toInt).isSuccess => true + case WomInteger(_) => true + case WomArray(WomArrayType(WomStringType), elements) => + elements forall { value => + Try(value.valueString.toInt).isSuccess + } + case WomArray(WomArrayType(WomIntegerType), _) => true + case _ => false } override protected def missingValueMessage: String = s"Expecting $key" + diff --git a/backend/src/main/scala/cromwell/backend/validation/ReturnCode.scala b/backend/src/main/scala/cromwell/backend/validation/ReturnCode.scala deleted file mode 100644 index a1a29e05f11..00000000000 --- a/backend/src/main/scala/cromwell/backend/validation/ReturnCode.scala +++ /dev/null @@ -1,34 +0,0 @@ -package cromwell.backend.validation - -trait ReturnCode { - - /** - * Returns true if the call is a success based on the return code. - * - * @param returnCode Return code from the process / script. - * @return True if the call is a success. - */ - def continueFor(returnCode: Int): Boolean = - this match { - case ReturnCodeSet(returnCodes) => returnCodes.contains(returnCode) - } -} - -/** - * Continues only if the call/job return code is found in returnCodes. - * @param returnCodes Inclusive set of return codes that specify a job success. - */ -case class ReturnCodeSet(returnCodes: Set[Int]) extends ReturnCode { - override def toString = returnCodes match { - case single if single.size == 1 => returnCodes.head.toString - case multiple => s"[${multiple.mkString(",")}]" - } - - /** - * Returns true if the call is a success based on the return code. - * - * @param returnCode Return code from the process / script. - * @return True if the call is a success. - */ - override def continueFor(returnCode: Int): Boolean = super.continueFor(returnCode) -} diff --git a/backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala deleted file mode 100644 index 14605cca41d..00000000000 --- a/backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala +++ /dev/null @@ -1,43 +0,0 @@ -package cromwell.backend.validation - -import cats.data.Validated.{Invalid, Valid} -import cats.implicits.{catsSyntaxValidatedId, toTraverseOps} -import common.validation.ErrorOr.ErrorOr -import cromwell.backend.validation.RuntimeAttributesValidation.validateInt -import wom.types.{WomArrayType, WomIntegerType, WomStringType, WomType} -import wom.values.{WomArray, WomInteger, WomString, WomValue} - -import scala.util.Try - -/** - * Validates the `continueOnReturnCode` and `returnCodes` runtime attributes, since they share the functionality of - * being a Set of Integers. - */ -trait ReturnCodeValidation extends RuntimeAttributesValidation[ReturnCode] { - override def key: String - override def coercion: Iterable[WomType] - override def validateValue: PartialFunction[WomValue, ErrorOr[ReturnCode]] = { - case WomString(value) if Try(value.toInt).isSuccess => ReturnCodeSet(Set(value.toInt)).validNel - case WomInteger(value) => ReturnCodeSet(Set(value)).validNel - case value @ WomArray(_, seq) => - val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int] - errorOrInts match { - case Valid(ints) => ReturnCodeSet(ints.toSet).validNel - case Invalid(_) => invalidValueFailure(value) - } - case value => invalidValueFailure(value) - } - - override def validateExpression: PartialFunction[WomValue, Boolean] = { - case WomString(value) if Try(value.toInt).isSuccess => true - case WomInteger(_) => true - case WomArray(WomArrayType(WomStringType), elements) => - elements forall { value => - Try(value.valueString.toInt).isSuccess - } - case WomArray(WomArrayType(WomIntegerType), _) => true - case _ => false - } - - override def usedInCallCaching: Boolean = true -} diff --git a/backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala b/backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala deleted file mode 100644 index 5cef8346ad1..00000000000 --- a/backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala +++ /dev/null @@ -1,33 +0,0 @@ -package cromwell.backend.validation - -import wom.types._ - -object ReturnCodes { - val validWdlTypes = Set[WomType](WomArrayType(WomIntegerType), WomStringType, WomIntegerType) -} - -/** - * Decides if a call/job continues upon a specific return code. - */ -sealed trait ReturnCodes extends ReturnCode { - - /** - * Returns true if the call is a success based on the return code. - * - * @param returnCode Return code from the process / script. - * @return True if the call is a success. - */ - final override def continueFor(returnCode: Int): Boolean = - this match { - case ReturnCodesString(continue) => continue.equals("*") || returnCode == 0 - case _ => super.continueFor(returnCode) - } -} - -/** - * Continues based on a string, if "*" all return codes continue. - * @param returnCode If "*", all return codes are valid for continuing. - */ -case class ReturnCodesString(returnCode: String) extends ReturnCodes { - override def toString = returnCode -} diff --git a/backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala deleted file mode 100644 index c3e78443eb4..00000000000 --- a/backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala +++ /dev/null @@ -1,51 +0,0 @@ -package cromwell.backend.validation - -import cats.implicits.catsSyntaxValidatedId -import com.typesafe.config.Config -import common.validation.ErrorOr.ErrorOr -import wom.RuntimeAttributesKeys -import wom.types.WomType -import wom.values.{WomInteger, WomString, WomValue} - -/** - * Validates the "returnCodes" runtime attribute as an Integer, returning the value as an `Int`, or as - * an array of Integers, returning the value as Array[Int], or "*", returning the value as a String. - * - * `default` a hardcoded default WomValue for returnCodes. - * - * `configDefaultWdlValue` returns the value of the attribute as specified by the - * reference.conf file, coerced into a WomValue. - */ -object ReturnCodesValidation { - lazy val instance: RuntimeAttributesValidation[ReturnCode] = new ReturnCodesValidation - lazy val optional: OptionalRuntimeAttributesValidation[ReturnCode] = instance.optional - def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ReturnCode] = - instance.withDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) - - def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = - instance.configDefaultWomValue(runtimeConfig) - def configDefaultWomValue(config: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(config) -} - -class ReturnCodesValidation extends ReturnCodeValidation { - - override def key: String = RuntimeAttributesKeys.ReturnCodesKey - - override def coercion: Set[WomType] = ReturnCodes.validWdlTypes - - override def validateValue: PartialFunction[WomValue, ErrorOr[ReturnCode]] = { - case WomString(value) if value.equals("*") => - ReturnCodesString(value).validNel - case value => super.validateValue(value) - } - - override def validateExpression: PartialFunction[WomValue, Boolean] = { - case WomString(value) if value.equals("*") => true - case value => super.validateExpression(value) - } - - override protected def missingValueMessage: String = s"Expecting $key" + - " runtime attribute to be either a String '*' or an Array[Int]" - - override def usedInCallCaching: Boolean = true -} diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala index 7cb513aad72..5fa52dac53a 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala @@ -34,12 +34,9 @@ object RuntimeAttributesValidation { def validateContinueOnReturnCode(value: Option[WomValue], onMissingKey: => ErrorOr[ContinueOnReturnCode] - ): ErrorOr[ReturnCode] = + ): ErrorOr[ContinueOnReturnCode] = validateWithValidation(value, ContinueOnReturnCodeValidation.instance, onMissingKey) - def validateReturnCodes(value: Option[WomValue], onMissingKey: => ErrorOr[ReturnCode]): ErrorOr[ReturnCode] = - validateWithValidation(value, ReturnCodesValidation.instance, onMissingKey) - def validateMemory(value: Option[WomValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = validateWithValidation(value, MemoryValidation.instance(), onMissingKey) @@ -375,8 +372,7 @@ trait RuntimeAttributesValidation[ValidatedType] { case Success(womValue) => validateExpression.applyOrElse(womValue, (_: Any) => false) case Failure(_) => true // If we can't evaluate it, we'll let it pass for now... } - case Some(womValue) => - validateExpression.applyOrElse(womValue, (_: Any) => false) + case Some(womValue) => validateExpression.applyOrElse(womValue, (_: Any) => false) } def validateOptionalWomExpression(womExpressionMaybe: Option[WomExpression]): Boolean = diff --git a/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala new file mode 100644 index 00000000000..e3ad710a4c4 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala @@ -0,0 +1,145 @@ +package cromwell.backend.validation + +import com.typesafe.config.Config +import common.validation.ErrorOr.ErrorOr +import cromwell.backend.RuntimeAttributeDefinition +import cromwell.backend.validation.RuntimeAttributesValidation.extractOption +import wom.types.WomType +import wom.values.{WomString, WomValue} + +object TwoKeyRuntimeAttributesValidation { + def withDefault[ValidatedType, DefaultValType]( + validation: TwoKeyRuntimeAttributesValidation[ValidatedType, DefaultValType], + default: WomValue + ): TwoKeyRuntimeAttributesValidation[ValidatedType, DefaultValType] = + new TwoKeyRuntimeAttributesValidation[ValidatedType, DefaultValType] { + override def key: String = validation.key + + override def altKey: String = validation.altKey + + override def defaultVal: DefaultValType = validation.defaultVal + + override def coercion: Iterable[WomType] = validation.coercion + + override protected def validateValue: PartialFunction[WomValue, ErrorOr[ValidatedType]] = + validation.validateValuePackagePrivate + + override protected def validateExpression: PartialFunction[WomValue, Boolean] = + validation.validateExpressionPackagePrivate + + override protected def invalidValueMessage(value: WomValue): String = + validation.invalidValueMessagePackagePrivate(value) + + override protected def missingValueMessage: String = s"Expecting $key runtime attribute to be a type in $coercion" + + override def usedInCallCaching: Boolean = validation.usedInCallCachingPackagePrivate + + override protected def staticDefaultOption = Option(default) + } + + /** + * Returns the value from the attributes matching one of the validation keys. If values are assigned to attribute + * matching both keys, the `key` field will override the `altKey` field on the `RuntimeAttributeValidation`. + * + * Do not use an optional validation as the type internal implementation will throw a `ClassCastException` due to the + * way values are located and auto-magically cast to the type of the `runtimeAttributesValidation`. + * + * @param runtimeAttributesValidation The typed validation to use. + * @param validatedRuntimeAttributes The values to search. + * @return The value matching the key. + * @throws ClassCastException if the validation is called on an optional validation. + */ + def extractTwoKeys[A, B](runtimeAttributesValidation: TwoKeyRuntimeAttributesValidation[A, B], + validatedRuntimeAttributes: ValidatedRuntimeAttributes + ): A = { + val key = runtimeAttributesValidation.key + val altKey = runtimeAttributesValidation.altKey + val primaryValue = extractOption(key, validatedRuntimeAttributes) + val altValue = extractOption(altKey, validatedRuntimeAttributes) + val value = + if ( + primaryValue.isEmpty || (primaryValue.contains( + runtimeAttributesValidation.defaultVal + ) && !altValue.isEmpty && !altValue.contains( + runtimeAttributesValidation.defaultVal + )) + ) altValue + else primaryValue + value match { + // NOTE: Some(innerValue) aka Some.unapply() throws a `ClassCastException` to `Nothing$` as it can't tell the type + case some: Some[_] => some.get.asInstanceOf[A] + case None => + throw new RuntimeException( + s"$key not found in runtime attributes ${validatedRuntimeAttributes.attributes.keys}" + ) + } + } +} + +trait TwoKeyRuntimeAttributesValidation[A, DefaultValType] extends RuntimeAttributesValidation[A] { + + /** + * Returns the alternate key of the runtime attribute. + * + * @return the alternate key of the runtime attribute. + */ + def altKey: String + + /** + * The default value of this runtime attribute. + * + * @return the default value of this runtime attribute. + */ + def defaultVal: DefaultValType + + /** + * Runs this validation on the value matching key. + * + * NOTE: The values passed to this method should already be evaluated instances of WomValue, and not WomExpression. + * + * @param values The full set of values. + * @return The error or valid value for this key. + */ + def validateAltKey(values: Map[String, WomValue]): ErrorOr[A] = + values.get(altKey) match { + case Some(value) => validateValue.applyOrElse(value, (_: Any) => invalidValueFailure(value)) + case None => validateNone + } + + /** + * Returns a version of this validation with the default value. + * + * @param womValue The default wdl value. + * @return The new version of this validation. + */ + final def makeDefault(womValue: WomValue): TwoKeyRuntimeAttributesValidation[A, DefaultValType] = + TwoKeyRuntimeAttributesValidation.withDefault(this, womValue) + + /** + * Returns the value of the default runtime attribute of a + * validation key as specified in the reference.conf. Given + * a value, this method coerces it into an optional + * WomValue. In case the value cannot be succesfully coerced + * the value is wrapped as a "BadDefaultAttributeValue" type that + * is failed downstream by the ValidatedRuntimeAttributesBuilder. + * + * @param optionalRuntimeConfig Optional default runtime attributes config of a particular backend. + * @return The new version of this validation. + */ + final def configDefault(optionalRuntimeConfig: Option[Config]): Option[WomValue] = + optionalRuntimeConfig collect { + case config if config.hasPath(key) || config.hasPath(altKey) => + val value = if (config.hasPath(key)) config.getValue(key).unwrapped() else config.getValue(altKey).unwrapped() + coercion collectFirst { + case womType if womType.coerceRawValue(value).isSuccess => womType.coerceRawValue(value).get + } getOrElse { + BadDefaultAttribute(WomString(value.toString)) + } + } + + /** + * Returns this as an instance of a runtime attribute definition. + */ + final lazy val altKeyRuntimeAttributeDefinition = + RuntimeAttributeDefinition(altKey, staticDefaultOption, usedInCallCaching) +} diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala index d1a6af6f58d..38dc38ccd27 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala @@ -10,6 +10,7 @@ import wom.expression.WomExpression import wom.types.WomType import wom.values.WomValue +import scala.collection.mutable.ListBuffer import scala.util.control.NoStackTrace final case class ValidatedRuntimeAttributes(attributes: Map[String, Any]) @@ -33,7 +34,15 @@ trait ValidatedRuntimeAttributesBuilder { /** * Returns a mapping of the validations: RuntimeAttributesValidation each converted to a RuntimeAttributeDefinition. */ - final lazy val definitions: Seq[RuntimeAttributeDefinition] = validations.map(_.runtimeAttributeDefinition) + final lazy val definitions: Seq[RuntimeAttributeDefinition] = + validations.map(_.runtimeAttributeDefinition) ++ validations + .map { + case value: TwoKeyRuntimeAttributesValidation[_, _] => + value.altKeyRuntimeAttributeDefinition + case _ => + null + } + .filterNot(x => x == null) /** * Returns validators suitable for BackendWorkflowInitializationActor.runtimeAttributeValidators. @@ -49,7 +58,14 @@ trait ValidatedRuntimeAttributesBuilder { def unsupportedKeys(keys: Seq[String]): Seq[String] = keys.diff(validationKeys) - private lazy val validationKeys = validations.map(_.key) + private lazy val validationKeys = validations.map(_.key) ++ validations + .map { + case value: TwoKeyRuntimeAttributesValidation[_, _] => + value.altKey + case _ => + null + } + .filterNot(x => x == null) def build(attrs: Map[String, WomValue], logger: Logger): ValidatedRuntimeAttributes = { RuntimeAttributesValidation.warnUnrecognized(attrs.keySet, validationKeys.toSet, logger) @@ -67,11 +83,23 @@ trait ValidatedRuntimeAttributesBuilder { } private def validate(values: Map[String, WomValue]): ErrorOr[ValidatedRuntimeAttributes] = { - val listOfKeysToErrorOrAnys: List[(String, ErrorOr[Any])] = - validations.map(validation => validation.key -> validation.validate(values)).toList + val listOfKeysToErrorOrAnys: ListBuffer[(String, ErrorOr[Any])] = ListBuffer() + + validations.foreach { validation => + listOfKeysToErrorOrAnys += validation.key -> validation.validate(values) + + validation match { + case twoKeyValidation: TwoKeyRuntimeAttributesValidation[_, _] => + if (values.contains(twoKeyValidation.altKey)) { + listOfKeysToErrorOrAnys += twoKeyValidation.altKey -> twoKeyValidation.validateAltKey(values) + } + case _ => + } + } - val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys map { case (key, errorOrAny) => - errorOrAny map { any => (key, any) } + val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys.toList map { + case (key, errorOrAny) => + errorOrAny map { any => (key, any) } } import cats.syntax.traverse._ diff --git a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala index c7cde397b69..1c04df95251 100644 --- a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala @@ -5,13 +5,7 @@ import _root_.wdl.draft2.model.types._ import akka.actor.ActorRef import akka.testkit.TestActorRef import com.typesafe.config.{Config, ConfigFactory} -import cromwell.backend.validation.{ - ContinueOnReturnCodeFlag, - ContinueOnReturnCodeValidation, - ReturnCodeSet, - ReturnCodesString, - ReturnCodesValidation -} +import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet, ContinueOnReturnCodeValidation} import cromwell.core.{TestKitSuite, WorkflowOptions} import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers @@ -43,16 +37,11 @@ class BackendWorkflowInitializationActorSpec val testContinueOnReturnCode: Option[WomValue] => Boolean = testPredicateBackendWorkflowInitializationActor.continueOnReturnCodePredicate(valueRequired = false) - val testReturnCodes: Option[WomValue] => Boolean = - testPredicateBackendWorkflowInitializationActor.returnCodesPredicate(valueRequired = false) - val optionalConfig: Option[Config] = Option(TestConfig.optionalRuntimeConfig) - it should "continueOnReturnCodePredicate and returnCodePredicate" in { + it should "continueOnReturnCodePredicate" in { testContinueOnReturnCode(None) should be(true) - testReturnCodes(None) should be(true) ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(None) should be(true) - ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(None) should be(true) val booleanRows = Table( "value", @@ -97,7 +86,7 @@ class BackendWorkflowInitializationActorSpec val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(value)) } @@ -112,7 +101,7 @@ class BackendWorkflowInitializationActorSpec val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(value)) } @@ -131,26 +120,15 @@ class BackendWorkflowInitializationActorSpec val womValue = WomInteger(value) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - testReturnCodes(Option(womValue)) should be(result) ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) - ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( - result - ) val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) - valid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) - - val returnCodesValid = - ReturnCodesValidation - .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) - returnCodesValid.isValid should be(result) - returnCodesValid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) + valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } forAll(integerRows) { value => @@ -160,25 +138,12 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) - - testReturnCodes(Option(womValue)) should be(result) - ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( - result - ) - val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) - valid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) - - val returnCodesValid = - ReturnCodesValidation - .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) - returnCodesValid.isValid should be(result) - returnCodesValid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) + valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } forAll(integerRows) { value => @@ -188,12 +153,6 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) - - testReturnCodes(Option(womValue)) should be(result) - ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( - result - ) - // NOTE: expressions are never valid to validate } @@ -204,25 +163,12 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) - - testReturnCodes(Option(womValue)) should be(result) - ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( - result - ) - val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) - valid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) - - val returnCodesValid = - ReturnCodesValidation - .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) - returnCodesValid.isValid should be(result) - returnCodesValid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) + valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } forAll(integerRows) { value => @@ -232,25 +178,12 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) - - testReturnCodes(Option(womValue)) should be(result) - ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( - result - ) - val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) - valid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) - - val returnCodesValid = - ReturnCodesValidation - .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) - returnCodesValid.isValid should be(result) - returnCodesValid.toEither.toOption.get should be(ReturnCodeSet(Set(value))) + valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } forAll(integerRows) { value => @@ -260,11 +193,6 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) - - testReturnCodes(Option(womValue)) should be(result) - ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( - result - ) // NOTE: expressions are never valid to validate } @@ -275,27 +203,22 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) - - testReturnCodes(Option(womValue)) should be(result) - ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( - result - ) // NOTE: expressions are never valid to validate } forAll(starRow) { value => val womValue = WomString(value) val result = true - testReturnCodes(Option(womValue)) should be(result) - ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + testContinueOnReturnCode(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) val valid = - ReturnCodesValidation + ContinueOnReturnCodeValidation .default(optionalConfig) .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) valid.isValid should be(result) - valid.toEither.toOption.get should be(ReturnCodesString(value)) + valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(true)) } forAll(invalidWdlValueRows) { womValue => @@ -304,28 +227,13 @@ class BackendWorkflowInitializationActorSpec ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( result ) - - testReturnCodes(Option(womValue)) should be(result) - ReturnCodesValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( - result - ) - val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.swap.toOption.get.toList should contain theSameElementsAs List( - "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" - ) - - val returnCodesValid = - ReturnCodesValidation - .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) - returnCodesValid.isValid should be(result) - returnCodesValid.toEither.swap.toOption.get.toList should contain theSameElementsAs List( - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) } @@ -359,7 +267,4 @@ class TestPredicateBackendWorkflowInitializationActor extends BackendWorkflowIni override def continueOnReturnCodePredicate(valueRequired: Boolean)(wdlExpressionMaybe: Option[WomValue]): Boolean = super.continueOnReturnCodePredicate(valueRequired)(wdlExpressionMaybe) - - override def returnCodesPredicate(valueRequired: Boolean)(wdlExpressionMaybe: Option[WomValue]): Boolean = - super.returnCodesPredicate(valueRequired)(wdlExpressionMaybe) } diff --git a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala index 818f29bbf51..3b8589765bb 100644 --- a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala @@ -40,7 +40,7 @@ class StandardValidatedRuntimeAttributesBuilderSpec """.stripMargin val defaultRuntimeAttributes: Map[String, Any] = - Map(DockerKey -> None, FailOnStderrKey -> false, ContinueOnReturnCodeKey -> ReturnCodeSet(Set(0))) + Map(DockerKey -> None, FailOnStderrKey -> false, ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(0))) def workflowOptionsWithDefaultRuntimeAttributes(defaults: Map[String, JsValue]): WorkflowOptions = WorkflowOptions(JsObject(Map("default_runtime_attributes" -> JsObject(defaults)))) @@ -128,7 +128,7 @@ class StandardValidatedRuntimeAttributesBuilderSpec "validate a valid continueOnReturnCode entry" in { val runtimeAttributes = Map("continueOnReturnCode" -> WomInteger(1)) val expectedRuntimeAttributes = - defaultRuntimeAttributes + (ContinueOnReturnCodeKey -> ReturnCodeSet(Set(1))) + defaultRuntimeAttributes + (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1))) assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -136,13 +136,13 @@ class StandardValidatedRuntimeAttributesBuilderSpec val runtimeAttributes = Map("continueOnReturnCode" -> WomString("value")) assertRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) } "use workflow options as default if continueOnReturnCode key is missing" in { val expectedRuntimeAttributes = defaultRuntimeAttributes + - (ContinueOnReturnCodeKey -> ReturnCodeSet(Set(1, 2))) + (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1, 2))) val workflowOptions = workflowOptionsWithDefaultRuntimeAttributes( Map(ContinueOnReturnCodeKey -> JsArray(Vector(JsNumber(1), JsNumber(2)))) ) @@ -156,7 +156,7 @@ class StandardValidatedRuntimeAttributesBuilderSpec "validate a valid returnCode entry" in { val runtimeAttributes = Map("returnCodes" -> WomInteger(1)) val expectedRuntimeAttributes = - defaultRuntimeAttributes + (ReturnCodesKey -> ReturnCodeSet(Set(1))) + defaultRuntimeAttributes + (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1))) assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -164,13 +164,13 @@ class StandardValidatedRuntimeAttributesBuilderSpec val runtimeAttributes = Map("returnCodes" -> WomString("value")) assertRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) } "use workflow options as default if returnCode key is missing" in { val expectedRuntimeAttributes = defaultRuntimeAttributes + - (ReturnCodesKey -> ReturnCodeSet(Set(1, 2))) + (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1, 2))) val workflowOptions = workflowOptionsWithDefaultRuntimeAttributes( Map(ReturnCodesKey -> JsArray(Vector(JsNumber(1), JsNumber(2)))) ) @@ -211,7 +211,9 @@ class StandardValidatedRuntimeAttributesBuilderSpec val docker = RuntimeAttributesValidation.extractOption(DockerValidation.instance, validatedRuntimeAttributes) val failOnStderr = RuntimeAttributesValidation.extract(FailOnStderrValidation.instance, validatedRuntimeAttributes) val continueOnReturnCode = - RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) + TwoKeyRuntimeAttributesValidation.extractTwoKeys(ContinueOnReturnCodeValidation.instance, + validatedRuntimeAttributes + ) docker should be(expectedRuntimeAttributes(DockerKey).asInstanceOf[Option[String]]) failOnStderr should be(expectedRuntimeAttributes(FailOnStderrKey).asInstanceOf[Boolean]) diff --git a/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala b/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala index 1ceb1666d46..e0ff75d3176 100644 --- a/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala @@ -34,7 +34,7 @@ class ContinueOnReturnCodeSpec extends AnyWordSpecLike with CromwellTimeoutSpec ) forAll(setTests) { (set, returnCode, expectedContinue) => - ReturnCodeSet(set).continueFor(returnCode) should be(expectedContinue) + ContinueOnReturnCodeSet(set).continueFor(returnCode) should be(expectedContinue) } } } diff --git a/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala b/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala index 7eb3694946a..3bd03fc036c 100644 --- a/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala @@ -12,24 +12,7 @@ class ReturnCodesSpec extends AnyWordSpecLike with CromwellTimeoutSpec with Matc val flagTests = Table(("flag", "returnCode", "expectedContinue"), ("*", 0, true), ("*", 1, true)) forAll(flagTests) { (flag, returnCode, expectedContinue) => - ReturnCodesString(flag).continueFor(returnCode) should be(expectedContinue) - } - } - - "continue on expected return code sets" in { - val setTests = Table( - ("set", "returnCode", "expectedContinue"), - (Set(0), 0, true), - (Set(0), 1, false), - (Set(1), 0, false), - (Set(1), 1, true), - (Set(0, 1), 0, true), - (Set(0, 1), 1, true), - (Set(0, 1), 2, false) - ) - - forAll(setTests) { (set, returnCode, expectedContinue) => - ReturnCodeSet(set).continueFor(returnCode) should be(expectedContinue) + ContinueOnReturnCodeFlag(true).continueFor(returnCode) should be(expectedContinue) } } } diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala index 284ae81efb9..e009a3adc89 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala @@ -169,7 +169,7 @@ class RuntimeAttributesValidationSpec "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel ) result match { - case Valid(x) => assert(x == ReturnCodeSet(Set(12))) + case Valid(x) => assert(x == ContinueOnReturnCodeSet(Set(12))) case Invalid(e) => fail(e.toList.mkString(" ")) } } @@ -184,7 +184,7 @@ class RuntimeAttributesValidationSpec case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert( - e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + e.head == "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) } } @@ -196,7 +196,7 @@ class RuntimeAttributesValidationSpec "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel ) result match { - case Valid(x) => assert(x == ReturnCodeSet(Set(1, 2))) + case Valid(x) => assert(x == ContinueOnReturnCodeSet(Set(1, 2))) case Invalid(e) => fail(e.toList.mkString(" ")) } } @@ -212,7 +212,7 @@ class RuntimeAttributesValidationSpec case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert( - e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + e.head == "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) } } @@ -230,71 +230,16 @@ class RuntimeAttributesValidationSpec "return success when tries to validate a valid returnCodes string entry" in { val returnCodesValue = Some(WomString("*")) - val result = RuntimeAttributesValidation.validateReturnCodes( + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( returnCodesValue, "Failed to get return code mandatory key from runtime attributes".invalidNel ) result match { - case Valid(x) => assert(x == ReturnCodesString("*")) - case Invalid(e) => fail(e.toList.mkString(" ")) - } - } - - "return success when tries to validate a returnCodes int entry" in { - val returnCodesValue = Some(WomInteger(12)) - val result = RuntimeAttributesValidation.validateReturnCodes( - returnCodesValue, - "Failed to get returnCodes mandatory key from runtime attributes".invalidNel - ) - result match { - case Valid(x) => assert(x == ReturnCodeSet(Set(12))) - case Invalid(e) => fail(e.toList.mkString(" ")) - } - } - - "return success when there is a valid integer array in returnCodes runtime attribute" in { - val returnCodesValue = Some(WomArray(WomArrayType(WomIntegerType), Seq(WomInteger(1), WomInteger(2)))) - val result = RuntimeAttributesValidation.validateReturnCodes( - returnCodesValue, - "Failed to get returnCode mandatory key from runtime attributes".invalidNel - ) - result match { - case Valid(x) => assert(x == ReturnCodeSet(Set(1, 2))) + case Valid(x) => assert(x == ContinueOnReturnCodeFlag(true)) case Invalid(e) => fail(e.toList.mkString(" ")) } } - "return failure when there is an invalid array in returnCodes runtime attribute" in { - val returnCodesValue = - Some(WomArray(WomArrayType(WomStringType), Seq(WomString("one"), WomString("two")))) - val result = RuntimeAttributesValidation.validateReturnCodes( - returnCodesValue, - "Failed to get returnCodes mandatory key from runtime attributes".invalidNel - ) - result match { - case Valid(_) => fail("A failure was expected.") - case Invalid(e) => - assert( - e.head == "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" - ) - } - } - - "return failure when there is an invalid returnCodes runtime attribute defined" in { - val returnCodesValue = Some(WomString("yes")) - val result = RuntimeAttributesValidation.validateReturnCodes( - returnCodesValue, - "Failed to get returnCodes mandatory key from runtime attributes".invalidNel - ) - result match { - case Valid(_) => fail("A failure was expected.") - case Invalid(e) => - assert( - e.head == "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" - ) - } - } - "return success when tries to validate a valid Integer memory entry" in { val expectedGb = 1 val memoryValue = Some(WomInteger(1 << 30)) diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala index 59c40cd6311..0ca650d2c04 100755 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala @@ -57,7 +57,6 @@ import scala.util.matching.Regex * @param queueArn the arn of the AWS Batch queue that the job will be submitted to * @param failOnStderr should the job fail if something is logged to `stderr` * @param continueOnReturnCode decides if a job continues on receiving a specific return code - * @param returnCodes decides if a job continues on receiving a specific return code * @param noAddress is there no address * @param scriptS3BucketName the s3 bucket where the execution command or script will be written and, from there, fetched into the container and executed * @param fileSystem the filesystem type, default is "s3" @@ -69,8 +68,7 @@ case class AwsBatchRuntimeAttributes(cpu: Int Refined Positive, dockerImage: String, queueArn: String, failOnStderr: Boolean, - continueOnReturnCode: ReturnCode, - returnCodes: ReturnCode, + continueOnReturnCode: ContinueOnReturnCode, noAddress: Boolean, scriptS3BucketName: String, fileSystem: String = "s3" @@ -104,9 +102,6 @@ object AwsBatchRuntimeAttributes { private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig) - private def returnCodesValidation(runtimeConfig: Option[Config]) = - ReturnCodesValidation.default(runtimeConfig) - private def disksValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Seq[AwsBatchVolume]] = DisksValidation .withDefault(DisksValidation.configDefaultWomValue(runtimeConfig) getOrElse DisksDefaultValue) @@ -189,14 +184,10 @@ object AwsBatchRuntimeAttributes { RuntimeAttributesValidation.extract(queueArnValidation(runtimeAttrsConfig), validatedRuntimeAttributes) val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(runtimeAttrsConfig), validatedRuntimeAttributes) - val continueOnReturnCode: ReturnCode = RuntimeAttributesValidation.extract( + val continueOnReturnCode: ContinueOnReturnCode = TwoKeyRuntimeAttributesValidation.extractTwoKeys( continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) - val returnCodes: ReturnCode = RuntimeAttributesValidation.extract( - returnCodesValidation(runtimeAttrsConfig), - validatedRuntimeAttributes - ) val noAddress: Boolean = RuntimeAttributesValidation.extract(noAddressValidation(runtimeAttrsConfig), validatedRuntimeAttributes) val scriptS3BucketName = fileSystem match { @@ -216,7 +207,6 @@ object AwsBatchRuntimeAttributes { queueArn, failOnStderr, continueOnReturnCode, - returnCodes, noAddress, scriptS3BucketName, fileSystem diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala index d4aa5232e4c..32d0cfab6bb 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchJobSpec.scala @@ -34,7 +34,7 @@ package cromwell.backend.impl.aws import common.collections.EnhancedCollections._ import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.backend.BackendSpec._ -import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ReturnCodeSet} +import cromwell.backend.validation.ContinueOnReturnCodeFlag import cromwell.core.TestKitSuite import cromwell.util.SampleWdl import eu.timepit.refined.api.Refined @@ -110,7 +110,6 @@ class AwsBatchJobSpec extends TestKitSuite with AnyFlatSpecLike with Matchers wi queueArn = "arn:aws:batch:us-east-1:123456789:job-queue/default-gwf-core", failOnStderr = true, continueOnReturnCode = ContinueOnReturnCodeFlag(false), - returnCodes = ReturnCodeSet(Set(0)), noAddress = false, scriptS3BucketName = "script-bucket", fileSystem = "s3" diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala index 2837e3cd23c..edc627afa83 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala @@ -35,7 +35,7 @@ import cats.data.NonEmptyList import common.assertion.CromwellTimeoutSpec import cromwell.backend.RuntimeAttributeDefinition import cromwell.backend.impl.aws.io.{AwsBatchVolume, AwsBatchWorkingDisk} -import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ReturnCodeSet, ReturnCodesString} +import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet} import cromwell.core.WorkflowOptions import eu.timepit.refined.numeric.Positive import eu.timepit.refined.refineMV @@ -67,8 +67,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "ubuntu:latest", "arn:aws:batch:us-east-1:111222333444:job-queue/job-queue", false, - ReturnCodeSet(Set(0)), - ReturnCodeSet(Set(0)), + ContinueOnReturnCodeSet(Set(0)), false, "my-stuff" ) @@ -81,8 +80,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "ubuntu:latest", "arn:aws:batch:us-east-1:111222333444:job-queue/job-queue", false, - ReturnCodeSet(Set(0)), - ReturnCodeSet(Set(0)), + ContinueOnReturnCodeSet(Set(0)), false, "", "local" @@ -164,7 +162,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "continueOnReturnCode" -> WomInteger(1) ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ReturnCodeSet(Set(1))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -183,7 +181,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "continueOnReturnCode" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ReturnCodeSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -193,7 +191,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "continueOnReturnCode" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ReturnCodeSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -213,7 +211,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "returnCodes" -> WomInteger(1) ) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodeSet(Set(1))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -222,7 +220,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "returnCodes" -> WomString("*") ) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesString("*")) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeFlag(true)) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -232,7 +230,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodeSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -242,7 +240,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout "scriptBucketName" -> WomString("my-stuff"), "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodeSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -253,7 +251,7 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout ) assertAwsBatchRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) } diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala index fac2e8d557e..5d7f87d8691 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala @@ -46,8 +46,7 @@ final case class GcpBatchRuntimeAttributes(cpu: Int Refined Positive, disks: Seq[GcpBatchAttachedDisk], dockerImage: String, failOnStderr: Boolean, - continueOnReturnCode: ReturnCode, - returnCodes: ReturnCode, + continueOnReturnCode: ContinueOnReturnCode, noAddress: Boolean, useDockerImageCache: Option[Boolean], checkpointFilename: Option[String] @@ -115,9 +114,6 @@ object GcpBatchRuntimeAttributes { private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig) - private def returnCodesValidation(runtimeConfig: Option[Config]) = - ReturnCodesValidation.default(runtimeConfig) - private def disksValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Seq[GcpBatchAttachedDisk]] = DisksValidation .withDefault(DisksValidation.configDefaultWomValue(runtimeConfig) getOrElse DisksDefaultValue) @@ -211,14 +207,10 @@ object GcpBatchRuntimeAttributes { val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes) val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(runtimeAttrsConfig), validatedRuntimeAttributes) - val continueOnReturnCode: ReturnCode = RuntimeAttributesValidation.extract( + val continueOnReturnCode: ContinueOnReturnCode = TwoKeyRuntimeAttributesValidation.extractTwoKeys( continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) - val returnCodes: ReturnCode = RuntimeAttributesValidation.extract( - returnCodesValidation(runtimeAttrsConfig), - validatedRuntimeAttributes - ) val noAddress: Boolean = RuntimeAttributesValidation.extract(noAddressValidation(runtimeAttrsConfig), validatedRuntimeAttributes) val zones: Vector[String] = RuntimeAttributesValidation.extract(ZonesValidation, validatedRuntimeAttributes) @@ -247,7 +239,6 @@ object GcpBatchRuntimeAttributes { docker, failOnStderr, continueOnReturnCode, - returnCodes, noAddress, useDockerImageCache, checkpointFileName diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala index 34024fe8f33..4a1e02cfd3d 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala @@ -3,7 +3,7 @@ package cromwell.backend.google.batch.models import cats.data.NonEmptyList import cromwell.backend.RuntimeAttributeDefinition import cromwell.backend.google.batch.models.GcpBatchTestConfig._ -import cromwell.backend.validation.ReturnCodeSet +import cromwell.backend.validation.ContinueOnReturnCodeSet //import cromwell.backend.google.batch.io.{DiskType, GcpBatchAttachedDisk} import cromwell.backend.google.batch.io.{DiskType, GcpBatchWorkingDisk} import cromwell.core.WorkflowOptions @@ -71,7 +71,7 @@ final class GcpBatchRuntimeAttributesSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomString("value")) assertBatchRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) } @@ -79,7 +79,7 @@ final class GcpBatchRuntimeAttributesSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("value")) assertBatchRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) } @@ -282,8 +282,7 @@ trait GcpBatchRuntimeAttributesSpecsMixin { disks = Vector(GcpBatchWorkingDisk(DiskType.SSD, 10)), dockerImage = "ubuntu:latest", failOnStderr = false, - continueOnReturnCode = ReturnCodeSet(Set(0)), - returnCodes = ReturnCodeSet(Set(0)), + continueOnReturnCode = ContinueOnReturnCodeSet(Set(0)), noAddress = false, useDockerImageCache = None, checkpointFilename = None diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala index a06d52c6b9e..df36cde3174 100644 --- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala +++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala @@ -49,8 +49,7 @@ final case class PipelinesApiRuntimeAttributes(cpu: Int Refined Positive, disks: Seq[PipelinesApiAttachedDisk], dockerImage: String, failOnStderr: Boolean, - continueOnReturnCode: ReturnCode, - returnCodes: ReturnCode, + continueOnReturnCode: ContinueOnReturnCode, noAddress: Boolean, googleLegacyMachineSelection: Boolean, useDockerImageCache: Option[Boolean], @@ -117,9 +116,6 @@ object PipelinesApiRuntimeAttributes { private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig) - private def returnCodesValidation(runtimeConfig: Option[Config]) = - ReturnCodesValidation.default(runtimeConfig) - private def disksValidation( runtimeConfig: Option[Config] ): RuntimeAttributesValidation[Seq[PipelinesApiAttachedDisk]] = DisksValidation @@ -223,14 +219,10 @@ object PipelinesApiRuntimeAttributes { val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes) val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(runtimeAttrsConfig), validatedRuntimeAttributes) - val continueOnReturnCode: ReturnCode = RuntimeAttributesValidation.extract( + val continueOnReturnCode: ContinueOnReturnCode = TwoKeyRuntimeAttributesValidation.extractTwoKeys( continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) - val returnCodes: ReturnCode = RuntimeAttributesValidation.extract( - returnCodesValidation(runtimeAttrsConfig), - validatedRuntimeAttributes - ) val noAddress: Boolean = RuntimeAttributesValidation.extract(noAddressValidation(runtimeAttrsConfig), validatedRuntimeAttributes) val useDockerImageCache: Option[Boolean] = RuntimeAttributesValidation.extractOption( @@ -250,7 +242,6 @@ object PipelinesApiRuntimeAttributes { docker, failOnStderr, continueOnReturnCode, - returnCodes, noAddress, googleLegacyMachineSelection, useDockerImageCache, diff --git a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala index 1267321cf6b..9a32e9365ae 100644 --- a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala +++ b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala @@ -4,7 +4,7 @@ import cats.data.NonEmptyList import cromwell.backend.RuntimeAttributeDefinition import cromwell.backend.google.pipelines.common.PipelinesApiTestConfig.{googleConfiguration, papiAttributes, _} import cromwell.backend.google.pipelines.common.io.{DiskType, PipelinesApiAttachedDisk, PipelinesApiWorkingDisk} -import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ReturnCodeSet, ReturnCodesString} +import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet} import cromwell.core.WorkflowOptions import eu.timepit.refined.refineMV import org.scalatest.TestSuite @@ -67,7 +67,7 @@ final class PipelinesApiRuntimeAttributesSpec "validate a valid continueOnReturnCode integer entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomInteger(1)) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ReturnCodeSet(Set(1))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -82,7 +82,7 @@ final class PipelinesApiRuntimeAttributesSpec Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ReturnCodeSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -91,7 +91,7 @@ final class PipelinesApiRuntimeAttributesSpec Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ReturnCodeSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -99,19 +99,19 @@ final class PipelinesApiRuntimeAttributesSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomString("value")) assertPapiRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) } "validate a valid returnCodes integer entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomInteger(1)) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodeSet(Set(1))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } "validate a valid returnCodes String entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("*")) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodesString("*")) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeFlag(true)) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -120,7 +120,7 @@ final class PipelinesApiRuntimeAttributesSpec Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodeSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -129,7 +129,7 @@ final class PipelinesApiRuntimeAttributesSpec Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) - val expectedRuntimeAttributes = expectedDefaults.copy(returnCodes = ReturnCodeSet(Set(1, 2))) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } @@ -137,7 +137,7 @@ final class PipelinesApiRuntimeAttributesSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("value")) assertPapiRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) } @@ -364,8 +364,7 @@ trait PipelinesApiRuntimeAttributesSpecsMixin { this: TestSuite => Vector(PipelinesApiWorkingDisk(DiskType.SSD, 10)), "ubuntu:latest", false, - ReturnCodeSet(Set(0)), - ReturnCodeSet(Set(0)), + ContinueOnReturnCodeSet(Set(0)), false, false, None, diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala index 409edd1adb9..e9ed89d1277 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala @@ -22,8 +22,7 @@ import wom.values._ import java.util.regex.Pattern -case class TesRuntimeAttributes(continueOnReturnCode: ReturnCode, - returnCodes: ReturnCode, +case class TesRuntimeAttributes(continueOnReturnCode: ContinueOnReturnCode, dockerImage: String, dockerWorkingDir: Option[String], failOnStderr: Boolean, @@ -49,9 +48,6 @@ object TesRuntimeAttributes { private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig) - private def returnCodesValidation(runtimeConfig: Option[Config]) = - ReturnCodesValidation.default(runtimeConfig) - private def diskSizeValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[MemorySize] = MemoryValidation.optional(DiskSizeKey) @@ -158,12 +154,10 @@ object TesRuntimeAttributes { val disk: Option[MemorySize] = detectDiskFormat(backendRuntimeConfig, validatedRuntimeAttributes) val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(backendRuntimeConfig), validatedRuntimeAttributes) - val continueOnReturnCode: ReturnCode = - RuntimeAttributesValidation.extract(continueOnReturnCodeValidation(backendRuntimeConfig), - validatedRuntimeAttributes + val continueOnReturnCode: ContinueOnReturnCode = + TwoKeyRuntimeAttributesValidation.extractTwoKeys(continueOnReturnCodeValidation(backendRuntimeConfig), + validatedRuntimeAttributes ) - val returnCodes: ReturnCode = - RuntimeAttributesValidation.extract(returnCodesValidation(backendRuntimeConfig), validatedRuntimeAttributes) val preemptible: Boolean = RuntimeAttributesValidation.extract(preemptibleValidation(backendRuntimeConfig), validatedRuntimeAttributes) val localizedSas: Option[String] = @@ -181,7 +175,6 @@ object TesRuntimeAttributes { diskSizeCompatValidation(backendRuntimeConfig), failOnStderrValidation(backendRuntimeConfig), continueOnReturnCodeValidation(backendRuntimeConfig), - returnCodesValidation(backendRuntimeConfig), preemptibleValidation(backendRuntimeConfig), localizedSasValidation ) @@ -193,7 +186,6 @@ object TesRuntimeAttributes { new TesRuntimeAttributes( continueOnReturnCode, - returnCodes, docker, dockerWorkingDir, failOnStderr, diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala index f3d563d2672..2adefbe40e4 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala @@ -1,7 +1,7 @@ package cromwell.backend.impl.tes import common.assertion.CromwellTimeoutSpec -import cromwell.backend.validation.{ReturnCodeSet, ReturnCodesString} +import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet} import cromwell.backend.{BackendConfigurationDescriptor, RuntimeAttributeDefinition, TestConfig} import cromwell.core.WorkflowOptions import eu.timepit.refined.numeric.Positive @@ -17,8 +17,7 @@ import wom.values._ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec with Matchers { val expectedDefaults = new TesRuntimeAttributes( - ReturnCodeSet(Set(0)), - ReturnCodeSet(Set(0)), + ContinueOnReturnCodeSet(Set(0)), "ubuntu:latest", None, false, @@ -130,7 +129,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec "validate a valid continueOnReturnCode entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomInteger(1)) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ReturnCodeSet(Set(1))) + expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } @@ -140,7 +139,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec "continueOnReturnCode" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ReturnCodeSet(Set(1, 2))) + expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } @@ -150,7 +149,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec "continueOnReturnCode" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ReturnCodeSet(Set(1, 2))) + expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } @@ -158,21 +157,21 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomString("value")) assertFailure( runtimeAttributes, - "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) } "validate a valid returnCodes int entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomInteger(1)) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodeSet(Set(1))) + expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } "validate a valid returnCodes String entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("*")) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodesString("*")) + expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeFlag(true)) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } @@ -182,7 +181,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) ) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodeSet(Set(1, 2))) + expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } @@ -192,7 +191,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) ) val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(returnCodes = ReturnCodeSet(Set(1, 2))) + expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) assertSuccess(runtimeAttributes, expectedRuntimeAttributes) } @@ -200,7 +199,7 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("value")) assertFailure( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" ) } diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala index ea6393f49a0..27e7a058d5d 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala @@ -2,7 +2,7 @@ package cromwell.backend.impl.tes import common.assertion.CromwellTimeoutSpec import common.mock.MockSugar -import cromwell.backend.validation.ReturnCodeSet +import cromwell.backend.validation.ContinueOnReturnCodeSet import cromwell.backend.{BackendSpec, BackendWorkflowDescriptor, TestConfig} import cromwell.core.{RootWorkflowId, WorkflowId, WorkflowOptions} import cromwell.core.labels.Labels @@ -18,8 +18,7 @@ import java.util.UUID class TesTaskSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with BackendSpec with MockSugar { val runtimeAttributes = new TesRuntimeAttributes( - ReturnCodeSet(Set(0)), - ReturnCodeSet(Set(0)), + ContinueOnReturnCodeSet(Set(0)), "ubuntu:latest", None, false, From 1e689fe1105eee11a63cb9f44e52440f2f63e687 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Mon, 1 Apr 2024 14:54:56 -0400 Subject: [PATCH 12/23] Fixed failing tests + more refactoring --- .../StandardAsyncExecutionActor.scala | 27 ++++--------------- .../ContinueOnReturnCodeValidation.scala | 11 ++++---- .../TwoKeyRuntimeAttributesValidation.scala | 20 +++++++------- .../ValidatedRuntimeAttributesBuilder.scala | 16 +++++------ ...ckendWorkflowInitializationActorSpec.scala | 3 ++- ...alidatedRuntimeAttributesBuilderSpec.scala | 6 +++-- .../RuntimeAttributesValidationSpec.scala | 6 +++-- .../failing_return_code.test | 2 +- .../aws/AwsBatchRuntimeAttributesSpec.scala | 6 +++-- .../GcpBatchRuntimeAttributesSpec.scala | 6 +++-- .../PipelinesApiRuntimeAttributesSpec.scala | 6 +++-- .../impl/tes/TesRuntimeAttributesSpec.scala | 6 +++-- 12 files changed, 54 insertions(+), 61 deletions(-) diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala index 2e29ecf9273..0078b07b2c7 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala @@ -37,7 +37,6 @@ import net.ceedubs.ficus.Ficus._ import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils import shapeless.Coproduct -import wom.RuntimeAttributesKeys.ReturnCodesKey import wom.callable.{AdHocValue, CommandTaskDefinition, ContainerizedInputExpression} import wom.expression.WomExpression import wom.graph.LocalName @@ -754,25 +753,9 @@ trait StandardAsyncExecutionActor * @return the behavior for continuing on the return code. */ lazy val continueOnReturnCode: ContinueOnReturnCode = - RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) - - /** - * Returns the behavior for continuing on the return code, obtained by converting `returnCodeContents` to an Int. - * - * @return the behavior for continuing on the return code. - */ - lazy val returnCodes: ContinueOnReturnCode = - RuntimeAttributesValidation.extract[ContinueOnReturnCode](ReturnCodesKey, validatedRuntimeAttributes) - - /** - * Returns the true behavior for continuing on the return code. If `returnCodes` runtime attribute is specified, - * then `continueOnReturnCode` attribute is ignored. If `returnCodes` is unspecified, then the value in - * `continueOnReturnCode` will be used. - */ - lazy val returnCode: ContinueOnReturnCode = - if (returnCodes.isInstanceOf[ContinueOnReturnCodeSet] && returnCodes.equals(ContinueOnReturnCodeSet(Set(0)))) - continueOnReturnCode - else returnCodes + TwoKeyRuntimeAttributesValidation.extractTwoKeys(ContinueOnReturnCodeValidation.instance, + validatedRuntimeAttributes + ) /** * Returns the max number of times that a failed job should be retried, obtained by converting `maxRetries` to an Int. @@ -1407,7 +1390,7 @@ trait StandardAsyncExecutionActor ) ) retryElseFail(executionHandle) - case Success(returnCodeAsInt) if returnCode.continueFor(returnCodeAsInt) => + case Success(returnCodeAsInt) if continueOnReturnCode.continueFor(returnCodeAsInt) => handleExecutionSuccess(status, oldHandle, returnCodeAsInt) // It's important that we check retryWithMoreMemory case before isAbort. RC could be 137 in either case; // if it was caused by OOM killer, want to handle as OOM and not job abort. @@ -1441,7 +1424,7 @@ trait StandardAsyncExecutionActor } else { tryReturnCodeAsInt match { case Success(returnCodeAsInt) - if outOfMemoryDetected && memoryRetryRequested && !returnCode.continueFor(returnCodeAsInt) => + if outOfMemoryDetected && memoryRetryRequested && !continueOnReturnCode.continueFor(returnCodeAsInt) => val executionHandle = Future.successful( FailedNonRetryableExecutionHandle( RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log), diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala index e55a36a838b..7a55ec29b6c 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala @@ -23,18 +23,17 @@ import scala.util.Try * `default` a validation with the default value specified by the reference.conf file. */ object ContinueOnReturnCodeValidation { - lazy val instance: TwoKeyRuntimeAttributesValidation[ContinueOnReturnCode, ContinueOnReturnCodeSet] = + lazy val instance: TwoKeyRuntimeAttributesValidation[ContinueOnReturnCode] = new ContinueOnReturnCodeValidation def default( runtimeConfig: Option[Config] - ): TwoKeyRuntimeAttributesValidation[ContinueOnReturnCode, ContinueOnReturnCodeSet] = + ): TwoKeyRuntimeAttributesValidation[ContinueOnReturnCode] = instance.makeDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = instance.configDefault(runtimeConfig) } -class ContinueOnReturnCodeValidation - extends TwoKeyRuntimeAttributesValidation[ContinueOnReturnCode, ContinueOnReturnCodeSet] { +class ContinueOnReturnCodeValidation extends TwoKeyRuntimeAttributesValidation[ContinueOnReturnCode] { override def key: String = RuntimeAttributesKeys.ReturnCodesKey @@ -74,7 +73,9 @@ class ContinueOnReturnCodeValidation } override protected def missingValueMessage: String = s"Expecting $key" + - " runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + " runtime attribute to be either a String '*' or an Array[Int]." + + s" Expecting $altKey" + + " runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" override def usedInCallCaching: Boolean = true } diff --git a/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala index e3ad710a4c4..a6c94863071 100644 --- a/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala @@ -8,16 +8,16 @@ import wom.types.WomType import wom.values.{WomString, WomValue} object TwoKeyRuntimeAttributesValidation { - def withDefault[ValidatedType, DefaultValType]( - validation: TwoKeyRuntimeAttributesValidation[ValidatedType, DefaultValType], + def withDefault[ValidatedType]( + validation: TwoKeyRuntimeAttributesValidation[ValidatedType], default: WomValue - ): TwoKeyRuntimeAttributesValidation[ValidatedType, DefaultValType] = - new TwoKeyRuntimeAttributesValidation[ValidatedType, DefaultValType] { + ): TwoKeyRuntimeAttributesValidation[ValidatedType] = + new TwoKeyRuntimeAttributesValidation[ValidatedType] { override def key: String = validation.key override def altKey: String = validation.altKey - override def defaultVal: DefaultValType = validation.defaultVal + override def defaultVal: ValidatedType = validation.defaultVal override def coercion: Iterable[WomType] = validation.coercion @@ -49,8 +49,8 @@ object TwoKeyRuntimeAttributesValidation { * @return The value matching the key. * @throws ClassCastException if the validation is called on an optional validation. */ - def extractTwoKeys[A, B](runtimeAttributesValidation: TwoKeyRuntimeAttributesValidation[A, B], - validatedRuntimeAttributes: ValidatedRuntimeAttributes + def extractTwoKeys[A](runtimeAttributesValidation: TwoKeyRuntimeAttributesValidation[A], + validatedRuntimeAttributes: ValidatedRuntimeAttributes ): A = { val key = runtimeAttributesValidation.key val altKey = runtimeAttributesValidation.altKey @@ -76,7 +76,7 @@ object TwoKeyRuntimeAttributesValidation { } } -trait TwoKeyRuntimeAttributesValidation[A, DefaultValType] extends RuntimeAttributesValidation[A] { +trait TwoKeyRuntimeAttributesValidation[A] extends RuntimeAttributesValidation[A] { /** * Returns the alternate key of the runtime attribute. @@ -90,7 +90,7 @@ trait TwoKeyRuntimeAttributesValidation[A, DefaultValType] extends RuntimeAttrib * * @return the default value of this runtime attribute. */ - def defaultVal: DefaultValType + def defaultVal: A /** * Runs this validation on the value matching key. @@ -112,7 +112,7 @@ trait TwoKeyRuntimeAttributesValidation[A, DefaultValType] extends RuntimeAttrib * @param womValue The default wdl value. * @return The new version of this validation. */ - final def makeDefault(womValue: WomValue): TwoKeyRuntimeAttributesValidation[A, DefaultValType] = + final def makeDefault(womValue: WomValue): TwoKeyRuntimeAttributesValidation[A] = TwoKeyRuntimeAttributesValidation.withDefault(this, womValue) /** diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala index 38dc38ccd27..86456e1d9c0 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala @@ -35,14 +35,10 @@ trait ValidatedRuntimeAttributesBuilder { * Returns a mapping of the validations: RuntimeAttributesValidation each converted to a RuntimeAttributeDefinition. */ final lazy val definitions: Seq[RuntimeAttributeDefinition] = - validations.map(_.runtimeAttributeDefinition) ++ validations - .map { - case value: TwoKeyRuntimeAttributesValidation[_, _] => - value.altKeyRuntimeAttributeDefinition - case _ => - null - } - .filterNot(x => x == null) + validations.map(_.runtimeAttributeDefinition) ++ validations.flatMap { + case value: TwoKeyRuntimeAttributesValidation[_] => Seq(value.altKeyRuntimeAttributeDefinition) + case _ => Seq() + } /** * Returns validators suitable for BackendWorkflowInitializationActor.runtimeAttributeValidators. @@ -60,7 +56,7 @@ trait ValidatedRuntimeAttributesBuilder { private lazy val validationKeys = validations.map(_.key) ++ validations .map { - case value: TwoKeyRuntimeAttributesValidation[_, _] => + case value: TwoKeyRuntimeAttributesValidation[_] => value.altKey case _ => null @@ -89,7 +85,7 @@ trait ValidatedRuntimeAttributesBuilder { listOfKeysToErrorOrAnys += validation.key -> validation.validate(values) validation match { - case twoKeyValidation: TwoKeyRuntimeAttributesValidation[_, _] => + case twoKeyValidation: TwoKeyRuntimeAttributesValidation[_] => if (values.contains(twoKeyValidation.altKey)) { listOfKeysToErrorOrAnys += twoKeyValidation.altKey -> twoKeyValidation.validateAltKey(values) } diff --git a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala index 1c04df95251..65a351bd815 100644 --- a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala @@ -233,7 +233,8 @@ class BackendWorkflowInitializationActorSpec .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.swap.toOption.get.toList should contain theSameElementsAs List( - "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } diff --git a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala index 3b8589765bb..bd0e3608458 100644 --- a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala @@ -136,7 +136,8 @@ class StandardValidatedRuntimeAttributesBuilderSpec val runtimeAttributes = Map("continueOnReturnCode" -> WomString("value")) assertRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } @@ -164,7 +165,8 @@ class StandardValidatedRuntimeAttributesBuilderSpec val runtimeAttributes = Map("returnCodes" -> WomString("value")) assertRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala index e009a3adc89..6e1a7b3c6a3 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala @@ -184,7 +184,8 @@ class RuntimeAttributesValidationSpec case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert( - e.head == "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + e.head == "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } } @@ -212,7 +213,8 @@ class RuntimeAttributesValidationSpec case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert( - e.head == "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + e.head == "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } } diff --git a/centaur/src/main/resources/standardTestCases/failing_return_code.test b/centaur/src/main/resources/standardTestCases/failing_return_code.test index a5a9196b8b1..163715222cf 100644 --- a/centaur/src/main/resources/standardTestCases/failing_return_code.test +++ b/centaur/src/main/resources/standardTestCases/failing_return_code.test @@ -8,4 +8,4 @@ files { metadata { workflowName: FailingReturnCode status: Failed -} \ No newline at end of file +} diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala index edc627afa83..ce22a7bc143 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala @@ -202,7 +202,8 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout ) assertAwsBatchRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } @@ -251,7 +252,8 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout ) assertAwsBatchRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala index 4a1e02cfd3d..93aceea7b3b 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala @@ -71,7 +71,8 @@ final class GcpBatchRuntimeAttributesSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomString("value")) assertBatchRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } @@ -79,7 +80,8 @@ final class GcpBatchRuntimeAttributesSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("value")) assertBatchRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } diff --git a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala index 9a32e9365ae..b6188b4e46c 100644 --- a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala +++ b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala @@ -99,7 +99,8 @@ final class PipelinesApiRuntimeAttributesSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomString("value")) assertPapiRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } @@ -137,7 +138,8 @@ final class PipelinesApiRuntimeAttributesSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("value")) assertPapiRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala index 2adefbe40e4..4b64870b3ef 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala @@ -157,7 +157,8 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomString("value")) assertFailure( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } @@ -199,7 +200,8 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("value")) assertFailure( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" ) } From 4ca9ca5c4afe4357aa233ac1abf3327fd2947fb1 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Wed, 3 Apr 2024 16:52:59 -0400 Subject: [PATCH 13/23] Hopefully fixed failing tests --- .../ContinueOnReturnCodeValidation.scala | 2 +- .../ValidatedRuntimeAttributesBuilder.scala | 19 ++++++++++--------- .../CallCacheHashingJobActor.scala | 12 +++++++++++- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala index 7a55ec29b6c..aff29246ca4 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala @@ -60,9 +60,9 @@ class ContinueOnReturnCodeValidation extends TwoKeyRuntimeAttributesValidation[C override def validateExpression: PartialFunction[WomValue, Boolean] = { case WomBoolean(_) => true + case WomString(value) if Try(value.toInt).isSuccess => true case WomString(value) if Try(value.toBoolean).isSuccess => true case WomString(value) if value.equals("*") => true - case WomString(value) if Try(value.toInt).isSuccess => true case WomInteger(_) => true case WomArray(WomArrayType(WomStringType), elements) => elements forall { value => diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala index 86456e1d9c0..b00602717df 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala @@ -44,7 +44,12 @@ trait ValidatedRuntimeAttributesBuilder { * Returns validators suitable for BackendWorkflowInitializationActor.runtimeAttributeValidators. */ final lazy val validatorMap: Map[String, Option[WomExpression] => Boolean] = - validations.map(validation => validation.key -> validation.validateOptionalWomExpression _).toMap + validations.map(validation => validation.key -> validation.validateOptionalWomExpression _).toMap ++ validations + .flatMap { + case value: TwoKeyRuntimeAttributesValidation[_] => + Seq(value.altKey -> value.validateOptionalWomExpression _).toMap + case _ => Seq() + } /** * Returns a map of coercions suitable for RuntimeAttributesDefault.workflowOptionsDefault. @@ -54,14 +59,10 @@ trait ValidatedRuntimeAttributesBuilder { def unsupportedKeys(keys: Seq[String]): Seq[String] = keys.diff(validationKeys) - private lazy val validationKeys = validations.map(_.key) ++ validations - .map { - case value: TwoKeyRuntimeAttributesValidation[_] => - value.altKey - case _ => - null - } - .filterNot(x => x == null) + private lazy val validationKeys = validations.map(_.key) ++ validations.flatMap { + case value: TwoKeyRuntimeAttributesValidation[_] => Seq(value.altKey) + case _ => Seq() + } def build(attrs: Map[String, WomValue], logger: Logger): ValidatedRuntimeAttributes = { RuntimeAttributesValidation.warnUnrecognized(attrs.keySet, validationKeys.toSet, logger) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala index ff36d6deb2f..718ea83609a 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala @@ -163,6 +163,8 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, val outputCountHash = HashResult(HashKey("output count"), jobDescriptor.taskCall.callable.outputs.size.toString.md5HashValue) + var returnCodeHashResult: HashResult = null + val runtimeAttributeHashes = runtimeAttributeDefinitions map { definition => jobDescriptor.runtimeAttributes.get(definition.name) match { case Some(_) @@ -171,6 +173,12 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, callCachingEligible.dockerHash.get.md5HashValue ) case Some(womValue) => + if (definition.name == "returnCodes") { + returnCodeHashResult = + HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), + womValue.valueString.md5HashValue + ) + } HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), womValue.valueString.md5HashValue ) @@ -181,6 +189,8 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, } } + val runtimeAttributeHashesEdited = runtimeAttributeHashes.removedAll(Set(returnCodeHashResult)) + val inputHashResults = nonFileInputs map { case WomValueSimpleton(name, value) => val womTypeHashKeyString = value.womType.toHashKeyString log.debug("Hashing input expression as {} {}", womTypeHashKeyString, name) @@ -204,7 +214,7 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, backendNameHash, inputCountHash, outputCountHash - ) ++ runtimeAttributeHashes ++ inputHashResults ++ outputExpressionHashResults + ) ++ runtimeAttributeHashesEdited ++ inputHashResults ++ outputExpressionHashResults } private[callcaching] def makeFileHashingActor() = { From 3f96f9df7923226d8a7c422596d8286ccc959719 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Fri, 5 Apr 2024 15:40:17 -0400 Subject: [PATCH 14/23] Refactor --- .github/workflows/integration_tests.yml | 8 -- .../StandardAsyncExecutionActor.scala | 4 +- .../ContinueOnReturnCodeValidation.scala | 20 ++--- .../TwoKeyRuntimeAttributesValidation.scala | 80 ------------------- .../ValidatedRuntimeAttributesBuilder.scala | 27 +------ ...ckendWorkflowInitializationActorSpec.scala | 16 ++-- ...alidatedRuntimeAttributesBuilderSpec.scala | 34 +------- ...turn_codes_and_continue_on_return_code.wdl | 6 +- .../CallCacheHashingJobActor.scala | 12 +-- .../impl/aws/AwsBatchRuntimeAttributes.scala | 2 +- .../aws/AwsBatchRuntimeAttributesSpec.scala | 62 -------------- .../models/GcpBatchRuntimeAttributes.scala | 2 +- ...tchAsyncBackendJobExecutionActorSpec.scala | 1 - .../GcpBatchRuntimeAttributesSpec.scala | 9 --- .../PipelinesApiRuntimeAttributes.scala | 2 +- ...ApiAsyncBackendJobExecutionActorSpec.scala | 1 - .../PipelinesApiRuntimeAttributesSpec.scala | 39 --------- .../impl/tes/TesRuntimeAttributes.scala | 4 +- .../impl/tes/TesRuntimeAttributesSpec.scala | 45 +---------- ...DefinitionElementToWomTaskDefinition.scala | 38 ++++++++- 20 files changed, 67 insertions(+), 345 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index caf88d30dd1..61ed3c5af41 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -96,14 +96,6 @@ jobs: - uses: ./.github/set_up_cromwell_action #This github action will set up git-secrets, caching, java, and sbt. with: cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} - # If a build step fails, activate SSH and idle for 30 minutes. - - name: Setup tmate session - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - timeout-minutes: 90 - with: - limit-access-to-actor: true - detached: true #This script bascially just looks up another script to run, assuming that the other script's filename is: #src/ci/bin/test${BUILD_TYPE}.sh. The first letter of the BUILD_TYPE is automatically capitalized when looking. - name: Run Integration Test diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala index 0078b07b2c7..bbb796220c4 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala @@ -753,9 +753,7 @@ trait StandardAsyncExecutionActor * @return the behavior for continuing on the return code. */ lazy val continueOnReturnCode: ContinueOnReturnCode = - TwoKeyRuntimeAttributesValidation.extractTwoKeys(ContinueOnReturnCodeValidation.instance, - validatedRuntimeAttributes - ) + RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) /** * Returns the max number of times that a failed job should be retried, obtained by converting `maxRetries` to an Int. diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala index aff29246ca4..00de322089c 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala @@ -23,23 +23,19 @@ import scala.util.Try * `default` a validation with the default value specified by the reference.conf file. */ object ContinueOnReturnCodeValidation { - lazy val instance: TwoKeyRuntimeAttributesValidation[ContinueOnReturnCode] = + lazy val instance: RuntimeAttributesValidation[ContinueOnReturnCode] = new ContinueOnReturnCodeValidation def default( runtimeConfig: Option[Config] - ): TwoKeyRuntimeAttributesValidation[ContinueOnReturnCode] = - instance.makeDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) + ): RuntimeAttributesValidation[ContinueOnReturnCode] = + instance.withDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = - instance.configDefault(runtimeConfig) + instance.configDefaultWomValue(runtimeConfig) } -class ContinueOnReturnCodeValidation extends TwoKeyRuntimeAttributesValidation[ContinueOnReturnCode] { +class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[ContinueOnReturnCode] { - override def key: String = RuntimeAttributesKeys.ReturnCodesKey - - override def altKey: String = RuntimeAttributesKeys.ContinueOnReturnCodeKey - - override def defaultVal: ContinueOnReturnCodeSet = ContinueOnReturnCodeSet(Set(0)) + override def key: String = RuntimeAttributesKeys.ContinueOnReturnCodeKey override def coercion: Set[WomType] = ContinueOnReturnCode.validWdlTypes @@ -72,9 +68,9 @@ class ContinueOnReturnCodeValidation extends TwoKeyRuntimeAttributesValidation[C case _ => false } - override protected def missingValueMessage: String = s"Expecting $key" + + override protected def missingValueMessage: String = "Expecting returnCodes" + " runtime attribute to be either a String '*' or an Array[Int]." + - s" Expecting $altKey" + + " Expecting continueOnReturnCode" + " runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" override def usedInCallCaching: Boolean = true diff --git a/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala index a6c94863071..25c326ecfcc 100644 --- a/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala @@ -3,79 +3,8 @@ package cromwell.backend.validation import com.typesafe.config.Config import common.validation.ErrorOr.ErrorOr import cromwell.backend.RuntimeAttributeDefinition -import cromwell.backend.validation.RuntimeAttributesValidation.extractOption -import wom.types.WomType import wom.values.{WomString, WomValue} -object TwoKeyRuntimeAttributesValidation { - def withDefault[ValidatedType]( - validation: TwoKeyRuntimeAttributesValidation[ValidatedType], - default: WomValue - ): TwoKeyRuntimeAttributesValidation[ValidatedType] = - new TwoKeyRuntimeAttributesValidation[ValidatedType] { - override def key: String = validation.key - - override def altKey: String = validation.altKey - - override def defaultVal: ValidatedType = validation.defaultVal - - override def coercion: Iterable[WomType] = validation.coercion - - override protected def validateValue: PartialFunction[WomValue, ErrorOr[ValidatedType]] = - validation.validateValuePackagePrivate - - override protected def validateExpression: PartialFunction[WomValue, Boolean] = - validation.validateExpressionPackagePrivate - - override protected def invalidValueMessage(value: WomValue): String = - validation.invalidValueMessagePackagePrivate(value) - - override protected def missingValueMessage: String = s"Expecting $key runtime attribute to be a type in $coercion" - - override def usedInCallCaching: Boolean = validation.usedInCallCachingPackagePrivate - - override protected def staticDefaultOption = Option(default) - } - - /** - * Returns the value from the attributes matching one of the validation keys. If values are assigned to attribute - * matching both keys, the `key` field will override the `altKey` field on the `RuntimeAttributeValidation`. - * - * Do not use an optional validation as the type internal implementation will throw a `ClassCastException` due to the - * way values are located and auto-magically cast to the type of the `runtimeAttributesValidation`. - * - * @param runtimeAttributesValidation The typed validation to use. - * @param validatedRuntimeAttributes The values to search. - * @return The value matching the key. - * @throws ClassCastException if the validation is called on an optional validation. - */ - def extractTwoKeys[A](runtimeAttributesValidation: TwoKeyRuntimeAttributesValidation[A], - validatedRuntimeAttributes: ValidatedRuntimeAttributes - ): A = { - val key = runtimeAttributesValidation.key - val altKey = runtimeAttributesValidation.altKey - val primaryValue = extractOption(key, validatedRuntimeAttributes) - val altValue = extractOption(altKey, validatedRuntimeAttributes) - val value = - if ( - primaryValue.isEmpty || (primaryValue.contains( - runtimeAttributesValidation.defaultVal - ) && !altValue.isEmpty && !altValue.contains( - runtimeAttributesValidation.defaultVal - )) - ) altValue - else primaryValue - value match { - // NOTE: Some(innerValue) aka Some.unapply() throws a `ClassCastException` to `Nothing$` as it can't tell the type - case some: Some[_] => some.get.asInstanceOf[A] - case None => - throw new RuntimeException( - s"$key not found in runtime attributes ${validatedRuntimeAttributes.attributes.keys}" - ) - } - } -} - trait TwoKeyRuntimeAttributesValidation[A] extends RuntimeAttributesValidation[A] { /** @@ -106,15 +35,6 @@ trait TwoKeyRuntimeAttributesValidation[A] extends RuntimeAttributesValidation[A case None => validateNone } - /** - * Returns a version of this validation with the default value. - * - * @param womValue The default wdl value. - * @return The new version of this validation. - */ - final def makeDefault(womValue: WomValue): TwoKeyRuntimeAttributesValidation[A] = - TwoKeyRuntimeAttributesValidation.withDefault(this, womValue) - /** * Returns the value of the default runtime attribute of a * validation key as specified in the reference.conf. Given diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala index b00602717df..63f5c5bc387 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala @@ -10,7 +10,6 @@ import wom.expression.WomExpression import wom.types.WomType import wom.values.WomValue -import scala.collection.mutable.ListBuffer import scala.util.control.NoStackTrace final case class ValidatedRuntimeAttributes(attributes: Map[String, Any]) @@ -34,11 +33,7 @@ trait ValidatedRuntimeAttributesBuilder { /** * Returns a mapping of the validations: RuntimeAttributesValidation each converted to a RuntimeAttributeDefinition. */ - final lazy val definitions: Seq[RuntimeAttributeDefinition] = - validations.map(_.runtimeAttributeDefinition) ++ validations.flatMap { - case value: TwoKeyRuntimeAttributesValidation[_] => Seq(value.altKeyRuntimeAttributeDefinition) - case _ => Seq() - } + final lazy val definitions: Seq[RuntimeAttributeDefinition] = validations.map(_.runtimeAttributeDefinition) /** * Returns validators suitable for BackendWorkflowInitializationActor.runtimeAttributeValidators. @@ -59,10 +54,7 @@ trait ValidatedRuntimeAttributesBuilder { def unsupportedKeys(keys: Seq[String]): Seq[String] = keys.diff(validationKeys) - private lazy val validationKeys = validations.map(_.key) ++ validations.flatMap { - case value: TwoKeyRuntimeAttributesValidation[_] => Seq(value.altKey) - case _ => Seq() - } + private lazy val validationKeys = validations.map(_.key) def build(attrs: Map[String, WomValue], logger: Logger): ValidatedRuntimeAttributes = { RuntimeAttributesValidation.warnUnrecognized(attrs.keySet, validationKeys.toSet, logger) @@ -80,19 +72,8 @@ trait ValidatedRuntimeAttributesBuilder { } private def validate(values: Map[String, WomValue]): ErrorOr[ValidatedRuntimeAttributes] = { - val listOfKeysToErrorOrAnys: ListBuffer[(String, ErrorOr[Any])] = ListBuffer() - - validations.foreach { validation => - listOfKeysToErrorOrAnys += validation.key -> validation.validate(values) - - validation match { - case twoKeyValidation: TwoKeyRuntimeAttributesValidation[_] => - if (values.contains(twoKeyValidation.altKey)) { - listOfKeysToErrorOrAnys += twoKeyValidation.altKey -> twoKeyValidation.validateAltKey(values) - } - case _ => - } - } + val listOfKeysToErrorOrAnys: List[(String, ErrorOr[Any])] = + validations.map(validation => validation.key -> validation.validate(values)).toList val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys.toList map { case (key, errorOrAny) => diff --git a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala index 65a351bd815..f710a60730d 100644 --- a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala @@ -86,7 +86,7 @@ class BackendWorkflowInitializationActorSpec val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(value)) } @@ -101,7 +101,7 @@ class BackendWorkflowInitializationActorSpec val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(value)) } @@ -126,7 +126,7 @@ class BackendWorkflowInitializationActorSpec val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -141,7 +141,7 @@ class BackendWorkflowInitializationActorSpec val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -166,7 +166,7 @@ class BackendWorkflowInitializationActorSpec val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -181,7 +181,7 @@ class BackendWorkflowInitializationActorSpec val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -216,7 +216,7 @@ class BackendWorkflowInitializationActorSpec val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validate(Map(RuntimeAttributesKeys.ReturnCodesKey -> womValue)) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(true)) } @@ -230,7 +230,7 @@ class BackendWorkflowInitializationActorSpec val valid = ContinueOnReturnCodeValidation .default(optionalConfig) - .validateAltKey(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.swap.toOption.get.toList should contain theSameElementsAs List( "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + diff --git a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala index bd0e3608458..99eb7d86db8 100644 --- a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala @@ -153,36 +153,6 @@ class StandardValidatedRuntimeAttributesBuilderSpec workflowOptions = workflowOptions ) } - - "validate a valid returnCode entry" in { - val runtimeAttributes = Map("returnCodes" -> WomInteger(1)) - val expectedRuntimeAttributes = - defaultRuntimeAttributes + (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1))) - assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) - } - - "fail to validate an invalid returnCode entry" in { - val runtimeAttributes = Map("returnCodes" -> WomString("value")) - assertRuntimeAttributesFailedCreation( - runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" - ) - } - - "use workflow options as default if returnCode key is missing" in { - val expectedRuntimeAttributes = defaultRuntimeAttributes + - (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1, 2))) - val workflowOptions = workflowOptionsWithDefaultRuntimeAttributes( - Map(ReturnCodesKey -> JsArray(Vector(JsNumber(1), JsNumber(2)))) - ) - val runtimeAttributes = Map.empty[String, WomValue] - assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, - expectedRuntimeAttributes, - workflowOptions = workflowOptions - ) - } - } val defaultLogger: Logger = LoggerFactory.getLogger(classOf[StandardValidatedRuntimeAttributesBuilderSpec]) @@ -213,9 +183,7 @@ class StandardValidatedRuntimeAttributesBuilderSpec val docker = RuntimeAttributesValidation.extractOption(DockerValidation.instance, validatedRuntimeAttributes) val failOnStderr = RuntimeAttributesValidation.extract(FailOnStderrValidation.instance, validatedRuntimeAttributes) val continueOnReturnCode = - TwoKeyRuntimeAttributesValidation.extractTwoKeys(ContinueOnReturnCodeValidation.instance, - validatedRuntimeAttributes - ) + RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) docker should be(expectedRuntimeAttributes(DockerKey).asInstanceOf[Option[String]]) failOnStderr should be(expectedRuntimeAttributes(FailOnStderrKey).asInstanceOf[Boolean]) diff --git a/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl index 2c9b4418298..0a94b2fcff4 100644 --- a/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl +++ b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl @@ -18,7 +18,7 @@ task ReturnCodeContinueOnReturnCode1 { runtime { docker: "ubuntu:latest" returnCodes: [1] - continueOnReturnCodes: [0] + continueOnReturnCode: [0] } } @@ -34,7 +34,7 @@ task ReturnCodeContinueOnReturnCode2 { runtime { docker: "ubuntu:latest" returnCodes: [1] - continueOnReturnCodes: false + continueOnReturnCode: false } } @@ -50,6 +50,6 @@ task ReturnCodeContinueOnReturnCode3 { runtime { docker: "ubuntu:latest" returnCodes: [1, 4, 7] - continueOnReturnCodes: [1, 3, 5] + continueOnReturnCode: [1, 3, 5] } } \ No newline at end of file diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala index 718ea83609a..ff36d6deb2f 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala @@ -163,8 +163,6 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, val outputCountHash = HashResult(HashKey("output count"), jobDescriptor.taskCall.callable.outputs.size.toString.md5HashValue) - var returnCodeHashResult: HashResult = null - val runtimeAttributeHashes = runtimeAttributeDefinitions map { definition => jobDescriptor.runtimeAttributes.get(definition.name) match { case Some(_) @@ -173,12 +171,6 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, callCachingEligible.dockerHash.get.md5HashValue ) case Some(womValue) => - if (definition.name == "returnCodes") { - returnCodeHashResult = - HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), - womValue.valueString.md5HashValue - ) - } HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), womValue.valueString.md5HashValue ) @@ -189,8 +181,6 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, } } - val runtimeAttributeHashesEdited = runtimeAttributeHashes.removedAll(Set(returnCodeHashResult)) - val inputHashResults = nonFileInputs map { case WomValueSimpleton(name, value) => val womTypeHashKeyString = value.womType.toHashKeyString log.debug("Hashing input expression as {} {}", womTypeHashKeyString, name) @@ -214,7 +204,7 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, backendNameHash, inputCountHash, outputCountHash - ) ++ runtimeAttributeHashesEdited ++ inputHashResults ++ outputExpressionHashResults + ) ++ runtimeAttributeHashes ++ inputHashResults ++ outputExpressionHashResults } private[callcaching] def makeFileHashingActor() = { diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala index 0ca650d2c04..447f48ae2b9 100755 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala @@ -184,7 +184,7 @@ object AwsBatchRuntimeAttributes { RuntimeAttributesValidation.extract(queueArnValidation(runtimeAttrsConfig), validatedRuntimeAttributes) val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(runtimeAttrsConfig), validatedRuntimeAttributes) - val continueOnReturnCode: ContinueOnReturnCode = TwoKeyRuntimeAttributesValidation.extractTwoKeys( + val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract( continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala index ce22a7bc143..f99f494e78d 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala @@ -195,68 +195,6 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } - "fail to validate an invalid continueOnReturnCode entry" in { - val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), - "scriptBucketName" -> WomString("my-stuff"), - "continueOnReturnCode" -> WomString("value") - ) - assertAwsBatchRuntimeAttributesFailedCreation( - runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" - ) - } - - "validate a valid returnCodes integer entry" in { - val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), - "scriptBucketName" -> WomString("my-stuff"), - "returnCodes" -> WomInteger(1) - ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) - assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) - } - - "validate a valid returnCodes String entry" in { - val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), - "scriptBucketName" -> WomString("my-stuff"), - "returnCodes" -> WomString("*") - ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeFlag(true)) - assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) - } - - "validate a valid returnCodes array entry" in { - val runtimeAttributes = Map( - "docker" -> WomString("ubuntu:latest"), - "scriptBucketName" -> WomString("my-stuff"), - "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) - ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) - assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) - } - - "coerce then validate a valid returnCodes array entry" in { - val runtimeAttributes = Map( - "docker" -> WomString("ubuntu:latest"), - "scriptBucketName" -> WomString("my-stuff"), - "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) - ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) - assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) - } - - "fail to validate an invalid returnCode entry" in { - val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), - "scriptBucketName" -> WomString("my-stuff"), - "returnCodes" -> WomString("value") - ) - assertAwsBatchRuntimeAttributesFailedCreation( - runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" - ) - } - "validate a valid cpu entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "scriptBucketName" -> WomString("my-stuff"), "cpu" -> WomInteger(2)) diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala index 5d7f87d8691..ef9e2389d4b 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala @@ -207,7 +207,7 @@ object GcpBatchRuntimeAttributes { val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes) val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(runtimeAttrsConfig), validatedRuntimeAttributes) - val continueOnReturnCode: ContinueOnReturnCode = TwoKeyRuntimeAttributesValidation.extractTwoKeys( + val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract( continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala index 9cfea1dd925..f8d6e071b31 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala @@ -939,7 +939,6 @@ class GcpBatchAsyncBackendJobExecutionActorSpec "runtimeAttributes:disks" -> "local-disk 200 SSD", "runtimeAttributes:docker" -> "ubuntu:latest", "runtimeAttributes:failOnStderr" -> "false", - "runtimeAttributes:returnCodes" -> "0", "runtimeAttributes:memory" -> "2 GB", "runtimeAttributes:noAddress" -> "false", "runtimeAttributes:preemptible" -> "0", diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala index 93aceea7b3b..4aca1e1a680 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala @@ -76,15 +76,6 @@ final class GcpBatchRuntimeAttributesSpec ) } - "fail to validate an invalid returnCodes entry" in { - val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("value")) - assertBatchRuntimeAttributesFailedCreation( - runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" - ) - } - "validate a valid cpu entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "cpu" -> WomInteger(2)) val expectedRuntimeAttributes = expectedDefaults.copy(cpu = refineMV(2)) diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala index df36cde3174..a472007b611 100644 --- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala +++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala @@ -219,7 +219,7 @@ object PipelinesApiRuntimeAttributes { val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes) val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(runtimeAttrsConfig), validatedRuntimeAttributes) - val continueOnReturnCode: ContinueOnReturnCode = TwoKeyRuntimeAttributesValidation.extractTwoKeys( + val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract( continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes ) diff --git a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala index 992ac594f97..41e826a4dc2 100644 --- a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala @@ -1726,7 +1726,6 @@ class PipelinesApiAsyncBackendJobExecutionActorSpec "runtimeAttributes:disks" -> "local-disk 200 SSD", "runtimeAttributes:docker" -> "ubuntu:latest", "runtimeAttributes:failOnStderr" -> "false", - "runtimeAttributes:returnCodes" -> "0", "runtimeAttributes:memory" -> "2 GB", "runtimeAttributes:noAddress" -> "false", "runtimeAttributes:preemptible" -> "0", diff --git a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala index b6188b4e46c..86f4cecfc0e 100644 --- a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala +++ b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala @@ -104,45 +104,6 @@ final class PipelinesApiRuntimeAttributesSpec ) } - "validate a valid returnCodes integer entry" in { - val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomInteger(1)) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) - assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) - } - - "validate a valid returnCodes String entry" in { - val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("*")) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeFlag(true)) - assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) - } - - "validate a valid returnCodes array entry" in { - val runtimeAttributes = - Map("docker" -> WomString("ubuntu:latest"), - "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) - ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) - assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) - } - - "coerce then validate a valid returnCodes array entry" in { - val runtimeAttributes = - Map("docker" -> WomString("ubuntu:latest"), - "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) - ) - val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) - assertPapiRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) - } - - "fail to validate an invalid returnCodes entry" in { - val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("value")) - assertPapiRuntimeAttributesFailedCreation( - runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" - ) - } - "validate a valid cpu entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "cpu" -> WomInteger(2)) val expectedRuntimeAttributes = expectedDefaults.copy(cpu = refineMV(2)) diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala index e9ed89d1277..117e0ee79a7 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala @@ -155,8 +155,8 @@ object TesRuntimeAttributes { val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(backendRuntimeConfig), validatedRuntimeAttributes) val continueOnReturnCode: ContinueOnReturnCode = - TwoKeyRuntimeAttributesValidation.extractTwoKeys(continueOnReturnCodeValidation(backendRuntimeConfig), - validatedRuntimeAttributes + RuntimeAttributesValidation.extract(continueOnReturnCodeValidation(backendRuntimeConfig), + validatedRuntimeAttributes ) val preemptible: Boolean = RuntimeAttributesValidation.extract(preemptibleValidation(backendRuntimeConfig), validatedRuntimeAttributes) diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala index 4b64870b3ef..55f4a08abd7 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala @@ -1,7 +1,7 @@ package cromwell.backend.impl.tes import common.assertion.CromwellTimeoutSpec -import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet} +import cromwell.backend.validation.ContinueOnReturnCodeSet import cromwell.backend.{BackendConfigurationDescriptor, RuntimeAttributeDefinition, TestConfig} import cromwell.core.WorkflowOptions import eu.timepit.refined.numeric.Positive @@ -162,49 +162,6 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec ) } - "validate a valid returnCodes int entry" in { - val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomInteger(1)) - val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) - assertSuccess(runtimeAttributes, expectedRuntimeAttributes) - } - - "validate a valid returnCodes String entry" in { - val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("*")) - val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeFlag(true)) - assertSuccess(runtimeAttributes, expectedRuntimeAttributes) - } - - "validate a valid returnCodes array entry" in { - val runtimeAttributes = - Map("docker" -> WomString("ubuntu:latest"), - "returnCodes" -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))) - ) - val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) - assertSuccess(runtimeAttributes, expectedRuntimeAttributes) - } - - "coerce then validate a valid returnCodes array entry" in { - val runtimeAttributes = - Map("docker" -> WomString("ubuntu:latest"), - "returnCodes" -> WomArray(WomArrayType(WomStringType), List(WomString("1"), WomString("2"))) - ) - val expectedRuntimeAttributes = - expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) - assertSuccess(runtimeAttributes, expectedRuntimeAttributes) - } - - "fail to validate an invalid returnCodes entry" in { - val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "returnCodes" -> WomString("value")) - assertFailure( - runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" - ) - } - "validate a valid cpu entry" in assertSuccess( Map("docker" -> WomString("ubuntu:latest"), "cpu" -> WomInteger(2)), expectedDefaultsPlusUbuntuDocker.copy(cpu = Option(refineMV[Positive](2))) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala index 576ce210da5..530a71a3608 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala @@ -8,14 +8,20 @@ import cats.syntax.validated._ import cats.instances.list._ import common.validation.ErrorOr.{ErrorOr, _} import wdl.model.draft3.elements.CommandPartElement.{PlaceholderCommandPartElement, StringCommandPartElement} -import wdl.model.draft3.elements.ExpressionElement.{ArrayLiteral, IdentifierLookup, KvPair, SelectFirst} +import wdl.model.draft3.elements.ExpressionElement.{ + ArrayLiteral, + IdentifierLookup, + KvPair, + PrimitiveLiteralExpressionElement, + SelectFirst +} import wdl.model.draft3.elements._ import wdl.model.draft3.graph.{ExpressionValueConsumer, LinkedGraph} import wom.callable.Callable._ import wom.callable.{Callable, CallableTaskDefinition, MetaValueElement} import wom.expression.WomExpression import wom.types.{WomOptionalType, WomType} -import wom.{CommandPart, RuntimeAttributes} +import wom.{CommandPart, RuntimeAttributes, RuntimeAttributesKeys} import wdl.model.draft3.graph.ExpressionValueConsumer.ops._ import wdl.model.draft3.graph.expression.{FileEvaluator, TypeEvaluator, ValueEvaluator} import wdl.model.draft3.graph.expression.WomExpressionMaker.ops._ @@ -25,6 +31,7 @@ import wdl.transforms.base.wdlom2wom.expression.renaming.IdentifierLookupRenamer import wdl.transforms.base.wdlom2wom.expression.renaming.expressionEvaluator import wdl.model.draft3.graph.expression.WomTypeMaker.ops._ import wdl.transforms.base.linking.typemakers._ +import wom.values.WomInteger object TaskDefinitionElementToWomTaskDefinition extends Util { @@ -333,7 +340,32 @@ object TaskDefinitionElementToWomTaskDefinition extends Util { womExpression <- kvPair.value.makeWomExpression(linkedGraph.typeAliases, consumedValueLookup) } yield kvPair.key -> womExpression - attributes.runtimeAttributes.toList + val returnCodesAttribute = + attributes.runtimeAttributes.toList.find(pair => pair.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) + val continueOnReturnCodeAttribute = + attributes.runtimeAttributes.toList.find(pair => pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey)) + + val returnCodesNotUnique = + returnCodesAttribute.get.value + .equals(continueOnReturnCodeAttribute.get.value) || returnCodesAttribute.get.value.equals( + ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) + ) + + var editedAttributes = attributes.runtimeAttributes + + if (!returnCodesNotUnique) { + editedAttributes = attributes.runtimeAttributes.filterNot(attribute => + attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) + ) + editedAttributes = editedAttributes ++ Vector( + KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesAttribute.get.value) + ) + } + + editedAttributes = + editedAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) + + editedAttributes.toList .traverse(processSingleRuntimeAttribute) .map(atts => RuntimeAttributes(atts.toMap)) } From 021a42e454636112c5ba30518fed463c3f881dc0 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Fri, 5 Apr 2024 16:03:59 -0400 Subject: [PATCH 15/23] Cleanup --- .../TwoKeyRuntimeAttributesValidation.scala | 65 ------------------- .../ValidatedRuntimeAttributesBuilder.scala | 9 +-- .../aws/AwsBatchRuntimeAttributesSpec.scala | 13 ++++ 3 files changed, 15 insertions(+), 72 deletions(-) delete mode 100644 backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala diff --git a/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala deleted file mode 100644 index 25c326ecfcc..00000000000 --- a/backend/src/main/scala/cromwell/backend/validation/TwoKeyRuntimeAttributesValidation.scala +++ /dev/null @@ -1,65 +0,0 @@ -package cromwell.backend.validation - -import com.typesafe.config.Config -import common.validation.ErrorOr.ErrorOr -import cromwell.backend.RuntimeAttributeDefinition -import wom.values.{WomString, WomValue} - -trait TwoKeyRuntimeAttributesValidation[A] extends RuntimeAttributesValidation[A] { - - /** - * Returns the alternate key of the runtime attribute. - * - * @return the alternate key of the runtime attribute. - */ - def altKey: String - - /** - * The default value of this runtime attribute. - * - * @return the default value of this runtime attribute. - */ - def defaultVal: A - - /** - * Runs this validation on the value matching key. - * - * NOTE: The values passed to this method should already be evaluated instances of WomValue, and not WomExpression. - * - * @param values The full set of values. - * @return The error or valid value for this key. - */ - def validateAltKey(values: Map[String, WomValue]): ErrorOr[A] = - values.get(altKey) match { - case Some(value) => validateValue.applyOrElse(value, (_: Any) => invalidValueFailure(value)) - case None => validateNone - } - - /** - * Returns the value of the default runtime attribute of a - * validation key as specified in the reference.conf. Given - * a value, this method coerces it into an optional - * WomValue. In case the value cannot be succesfully coerced - * the value is wrapped as a "BadDefaultAttributeValue" type that - * is failed downstream by the ValidatedRuntimeAttributesBuilder. - * - * @param optionalRuntimeConfig Optional default runtime attributes config of a particular backend. - * @return The new version of this validation. - */ - final def configDefault(optionalRuntimeConfig: Option[Config]): Option[WomValue] = - optionalRuntimeConfig collect { - case config if config.hasPath(key) || config.hasPath(altKey) => - val value = if (config.hasPath(key)) config.getValue(key).unwrapped() else config.getValue(altKey).unwrapped() - coercion collectFirst { - case womType if womType.coerceRawValue(value).isSuccess => womType.coerceRawValue(value).get - } getOrElse { - BadDefaultAttribute(WomString(value.toString)) - } - } - - /** - * Returns this as an instance of a runtime attribute definition. - */ - final lazy val altKeyRuntimeAttributeDefinition = - RuntimeAttributeDefinition(altKey, staticDefaultOption, usedInCallCaching) -} diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala index 63f5c5bc387..3e8b1b3475a 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala @@ -39,12 +39,7 @@ trait ValidatedRuntimeAttributesBuilder { * Returns validators suitable for BackendWorkflowInitializationActor.runtimeAttributeValidators. */ final lazy val validatorMap: Map[String, Option[WomExpression] => Boolean] = - validations.map(validation => validation.key -> validation.validateOptionalWomExpression _).toMap ++ validations - .flatMap { - case value: TwoKeyRuntimeAttributesValidation[_] => - Seq(value.altKey -> value.validateOptionalWomExpression _).toMap - case _ => Seq() - } + validations.map(validation => validation.key -> validation.validateOptionalWomExpression _).toMap /** * Returns a map of coercions suitable for RuntimeAttributesDefault.workflowOptionsDefault. @@ -75,7 +70,7 @@ trait ValidatedRuntimeAttributesBuilder { val listOfKeysToErrorOrAnys: List[(String, ErrorOr[Any])] = validations.map(validation => validation.key -> validation.validate(values)).toList - val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys.toList map { + val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys map { case (key, errorOrAny) => errorOrAny map { any => (key, any) } } diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala index f99f494e78d..ca4a6425ef8 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala @@ -195,6 +195,19 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } + "fail to validate an invalid continueOnReturnCode entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), + "scriptBucketName" -> WomString("my-stuff"), + "continueOnReturnCode" -> WomString("value") + ) + assertAwsBatchRuntimeAttributesFailedCreation( + runtimeAttributes, + "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + + "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an " + + "Array[Int]" + ) + } + "validate a valid cpu entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "scriptBucketName" -> WomString("my-stuff"), "cpu" -> WomInteger(2)) From dba7c9eff947c04976555386956b5d637edde327 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Fri, 5 Apr 2024 16:06:39 -0400 Subject: [PATCH 16/23] scalafmt --- .../validation/ValidatedRuntimeAttributesBuilder.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala index 3e8b1b3475a..d1a6af6f58d 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala @@ -70,9 +70,8 @@ trait ValidatedRuntimeAttributesBuilder { val listOfKeysToErrorOrAnys: List[(String, ErrorOr[Any])] = validations.map(validation => validation.key -> validation.validate(values)).toList - val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys map { - case (key, errorOrAny) => - errorOrAny map { any => (key, any) } + val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys map { case (key, errorOrAny) => + errorOrAny map { any => (key, any) } } import cats.syntax.traverse._ From 461525c03a115e9ed5168267f29e940f54e325d3 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Mon, 8 Apr 2024 09:13:14 -0400 Subject: [PATCH 17/23] Fixed failing tests --- ...DefinitionElementToWomTaskDefinition.scala | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala index 530a71a3608..267b38fcfc8 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala @@ -345,21 +345,25 @@ object TaskDefinitionElementToWomTaskDefinition extends Util { val continueOnReturnCodeAttribute = attributes.runtimeAttributes.toList.find(pair => pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey)) - val returnCodesNotUnique = - returnCodesAttribute.get.value - .equals(continueOnReturnCodeAttribute.get.value) || returnCodesAttribute.get.value.equals( - ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) - ) - + val returnCodesGet = returnCodesAttribute.orNull + val continueOnReturnCodeGet = continueOnReturnCodeAttribute.orNull var editedAttributes = attributes.runtimeAttributes - if (!returnCodesNotUnique) { - editedAttributes = attributes.runtimeAttributes.filterNot(attribute => - attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) - ) - editedAttributes = editedAttributes ++ Vector( - KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesAttribute.get.value) - ) + if (returnCodesGet != null) { + val returnCodesNotUnique = + returnCodesGet.value + .equals(continueOnReturnCodeGet.value) || returnCodesGet.value.equals( + ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) + ) + + if (!returnCodesNotUnique) { + editedAttributes = attributes.runtimeAttributes.filterNot(attribute => + attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) + ) + editedAttributes = editedAttributes ++ Vector( + KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesAttribute.get.value) + ) + } } editedAttributes = From 30a126221eb41d5425b66a4e1ebb0f4d8bb51605 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Mon, 8 Apr 2024 13:27:14 -0400 Subject: [PATCH 18/23] Refactor and fix failing tests --- .../ContinueOnReturnCodeValidation.scala | 6 +- ...ckendWorkflowInitializationActorSpec.scala | 4 +- ...alidatedRuntimeAttributesBuilderSpec.scala | 4 +- .../RuntimeAttributesValidationSpec.scala | 8 +- ...turn_codes_and_continue_on_return_code.wdl | 4 +- .../return_codes/return_codes.wdl | 2 +- ...turn_codes_and_continue_on_return_code.wdl | 2 +- .../aws/AwsBatchRuntimeAttributesSpec.scala | 5 +- .../GcpBatchRuntimeAttributesSpec.scala | 4 +- .../PipelinesApiRuntimeAttributesSpec.scala | 4 +- .../impl/tes/TesRuntimeAttributesSpec.scala | 4 +- .../biscayne/wdlom2wom/package.scala | 123 +++++++++++++++++- .../cascades/wdlom2wom/package.scala | 123 +++++++++++++++++- ...DefinitionElementToWomTaskDefinition.scala | 59 +++------ 14 files changed, 279 insertions(+), 73 deletions(-) diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala index 00de322089c..7573909eb69 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala @@ -68,10 +68,8 @@ class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[Continu case _ => false } - override protected def missingValueMessage: String = "Expecting returnCodes" + - " runtime attribute to be either a String '*' or an Array[Int]." + - " Expecting continueOnReturnCode" + - " runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" + override protected def missingValueMessage: String = "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." override def usedInCallCaching: Boolean = true } diff --git a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala index f710a60730d..c4688d759f8 100644 --- a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala @@ -233,8 +233,8 @@ class BackendWorkflowInitializationActorSpec .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.swap.toOption.get.toList should contain theSameElementsAs List( - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." ) } diff --git a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala index 99eb7d86db8..35c57983e6e 100644 --- a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala @@ -136,8 +136,8 @@ class StandardValidatedRuntimeAttributesBuilderSpec val runtimeAttributes = Map("continueOnReturnCode" -> WomString("value")) assertRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." ) } diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala index 6e1a7b3c6a3..dafca35e1b6 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala @@ -184,8 +184,8 @@ class RuntimeAttributesValidationSpec case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert( - e.head == "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" + e.head == "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." ) } } @@ -213,8 +213,8 @@ class RuntimeAttributesValidationSpec case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert( - e.head == "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" + e.head == "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." ) } } diff --git a/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl index 3192880b1da..ea2233ab355 100644 --- a/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl +++ b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl @@ -16,6 +16,6 @@ task InvalidReturnCodeContinueOnReturnCode { runtime { docker: "ubuntu:latest" returnCodes: [5, 10, 15] - continueOnReturnCodes: [1] + continueOnReturnCode: [1] } -} \ No newline at end of file +} diff --git a/centaur/src/main/resources/standardTestCases/return_codes/return_codes.wdl b/centaur/src/main/resources/standardTestCases/return_codes/return_codes.wdl index d760a5c1f81..21e1da37650 100644 --- a/centaur/src/main/resources/standardTestCases/return_codes/return_codes.wdl +++ b/centaur/src/main/resources/standardTestCases/return_codes/return_codes.wdl @@ -66,4 +66,4 @@ task ReturnCodeString { docker: "ubuntu:latest" returnCodes: "*" } -} \ No newline at end of file +} diff --git a/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl index 0a94b2fcff4..41acb028221 100644 --- a/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl +++ b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl @@ -52,4 +52,4 @@ task ReturnCodeContinueOnReturnCode3 { returnCodes: [1, 4, 7] continueOnReturnCode: [1, 3, 5] } -} \ No newline at end of file +} diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala index ca4a6425ef8..f972f4b6f2e 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala @@ -202,9 +202,8 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout ) assertAwsBatchRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an " + - "Array[Int]" + "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." ) } diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala index 4aca1e1a680..ba68e169579 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala @@ -71,8 +71,8 @@ final class GcpBatchRuntimeAttributesSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomString("value")) assertBatchRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." ) } diff --git a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala index 86f4cecfc0e..542883743e2 100644 --- a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala +++ b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributesSpec.scala @@ -99,8 +99,8 @@ final class PipelinesApiRuntimeAttributesSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomString("value")) assertPapiRuntimeAttributesFailedCreation( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." ) } diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala index 55f4a08abd7..7189f64a660 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala @@ -157,8 +157,8 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "continueOnReturnCode" -> WomString("value")) assertFailure( runtimeAttributes, - "Expecting returnCodes runtime attribute to be either a String '*' or an Array[Int]. " + - "Expecting continueOnReturnCode runtime attribute to be a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." ) } diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala index ef1f719a00e..5fb11a1de10 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala @@ -1,9 +1,23 @@ package wdl.transforms.biscayne +import cats.implicits.{catsSyntaxTuple2Semigroupal, catsSyntaxValidatedId, toTraverseOps} import common.transforms.CheckedAtoB -import wdl.transforms.base.wdlom2wom.TaskDefinitionElementToWomTaskDefinition.TaskDefinitionElementToWomInputs +import common.validation.ErrorOr.{ErrorOr, ShortCircuitingFlatMapTuple2, _} +import wdl.model.draft3.elements.{ExpressionElement, RuntimeAttributesSectionElement} +import wdl.model.draft3.elements.ExpressionElement.{ArrayLiteral, KvPair, PrimitiveLiteralExpressionElement} +import wdl.model.draft3.graph.ExpressionValueConsumer +import wdl.model.draft3.graph.expression.{FileEvaluator, TypeEvaluator, ValueEvaluator} +import wdl.transforms.base.wdlom2wom.TaskDefinitionElementToWomTaskDefinition.{ + createTaskGraph, + eliminateInputDependencies, + expandLines, + processMetaSections, + validateParameterMetaEntries, + TaskDefinitionElementToWomInputs +} import wdl.transforms.base.wdlom2wom.WorkflowDefinitionElementToWomWorkflowDefinition.WorkflowDefinitionConvertInputs import wdl.transforms.base.wdlom2wom.{ + CommandPartElementToWomCommandPart, FileElementToWomBundle, FileElementToWomBundleInputs, TaskDefinitionElementToWomTaskDefinition, @@ -15,13 +29,118 @@ import wdl.transforms.biscayne.linking.expression.consumed._ import wdl.transforms.biscayne.linking.expression.files._ import wdl.transforms.biscayne.linking.expression.types._ import wdl.transforms.biscayne.linking.expression.values._ +import wom.values.WomInteger +import wom.{CommandPart, RuntimeAttributes, RuntimeAttributesKeys} package object wdlom2wom { val taskDefinitionElementToWomTaskDefinition: CheckedAtoB[TaskDefinitionElementToWomInputs, CallableTaskDefinition] = - CheckedAtoB.fromErrorOr(TaskDefinitionElementToWomTaskDefinition.convert) + CheckedAtoB.fromErrorOr(convert) val workflowDefinitionElementToWomWorkflowDefinition : CheckedAtoB[WorkflowDefinitionConvertInputs, WorkflowDefinition] = CheckedAtoB.fromErrorOr(WorkflowDefinitionElementToWomWorkflowDefinition.convert) val fileElementToWomBundle: CheckedAtoB[FileElementToWomBundleInputs, WomBundle] = CheckedAtoB.fromCheck(FileElementToWomBundle.convert) + + private def convert(b: TaskDefinitionElementToWomInputs)(implicit + expressionValueConsumer: ExpressionValueConsumer[ExpressionElement], + fileEvaluator: FileEvaluator[ExpressionElement], + typeEvaluator: TypeEvaluator[ExpressionElement], + valueEvaluator: ValueEvaluator[ExpressionElement] + ): ErrorOr[CallableTaskDefinition] = { + val a = eliminateInputDependencies(b)(expressionValueConsumer) + val inputElements = a.taskDefinitionElement.inputsSection.map(_.inputDeclarations).getOrElse(Seq.empty) + + val declarations = a.taskDefinitionElement.declarations + val outputElements = a.taskDefinitionElement.outputsSection.map(_.outputs).getOrElse(Seq.empty) + + val conversion = ( + createTaskGraph(inputElements, + declarations, + outputElements, + a.taskDefinitionElement.parameterMetaSection, + a.typeAliases + )(expressionValueConsumer, fileEvaluator, typeEvaluator, valueEvaluator), + validateParameterMetaEntries(a.taskDefinitionElement.parameterMetaSection, + a.taskDefinitionElement.inputsSection, + a.taskDefinitionElement.outputsSection + ) + ) flatMapN { (taskGraph, _) => + val validRuntimeAttributes: ErrorOr[RuntimeAttributes] = a.taskDefinitionElement.runtimeSection match { + case Some(attributeSection) => + val returnCodesAttribute = + attributeSection.runtimeAttributes.toList.find(pair => + pair.key.equals(RuntimeAttributesKeys.ReturnCodesKey) + ) + val continueOnReturnCodeAttribute = + attributeSection.runtimeAttributes.toList.find(pair => + pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) + ) + + val returnCodesGet = returnCodesAttribute.orNull + val continueOnReturnCodeGet = continueOnReturnCodeAttribute.orNull + var editedAttributes = attributeSection.runtimeAttributes + var returnCodesNotUnique = false + + if (returnCodesGet != null && continueOnReturnCodeGet != null) { + returnCodesNotUnique = returnCodesGet.value + .equals(continueOnReturnCodeGet.value) || returnCodesGet.value.equals( + ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) + ) + } + + if (returnCodesGet != null && !returnCodesNotUnique) { + editedAttributes = attributeSection.runtimeAttributes.filterNot(attribute => + attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) + ) + editedAttributes = editedAttributes ++ Vector( + KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesAttribute.get.value) + ) + } + + editedAttributes = + editedAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) + + TaskDefinitionElementToWomTaskDefinition.createRuntimeAttributes( + RuntimeAttributesSectionElement(editedAttributes), + taskGraph.linkedGraph + )( + expressionValueConsumer, + fileEvaluator, + typeEvaluator, + valueEvaluator + ) + case None => RuntimeAttributes(Map.empty).validNel + } + + val validCommand: ErrorOr[Seq[CommandPart]] = + expandLines(a.taskDefinitionElement.commandSection.parts).toList + .traverse { parts => + CommandPartElementToWomCommandPart.convert(parts, + taskGraph.linkedGraph.typeAliases, + taskGraph.linkedGraph.generatedHandles + )(expressionValueConsumer, fileEvaluator, typeEvaluator, valueEvaluator) + } + .map(_.toSeq) + + val (meta, parameterMeta) = + processMetaSections(a.taskDefinitionElement.metaSection, a.taskDefinitionElement.parameterMetaSection) + + (validRuntimeAttributes, validCommand) mapN { (runtime, command) => + CallableTaskDefinition( + a.taskDefinitionElement.name, + Function.const(command.validNel), + runtime, + meta, + parameterMeta, + taskGraph.outputs, + taskGraph.inputs, + Set.empty, + Map.empty, + sourceLocation = a.taskDefinitionElement.sourceLocation + ) + } + } + + conversion.contextualizeErrors(s"process task definition '${b.taskDefinitionElement.name}'") + } } diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala index 8933a00cb57..35f2fa8b8cc 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala @@ -1,9 +1,23 @@ package wdl.transforms.cascades +import cats.implicits.{catsSyntaxTuple2Semigroupal, catsSyntaxValidatedId, toTraverseOps} import common.transforms.CheckedAtoB -import wdl.transforms.base.wdlom2wom.TaskDefinitionElementToWomTaskDefinition.TaskDefinitionElementToWomInputs +import common.validation.ErrorOr.{ErrorOr, ShortCircuitingFlatMapTuple2, _} +import wdl.model.draft3.elements.ExpressionElement.{ArrayLiteral, KvPair, PrimitiveLiteralExpressionElement} +import wdl.model.draft3.elements.{ExpressionElement, RuntimeAttributesSectionElement} +import wdl.model.draft3.graph.ExpressionValueConsumer +import wdl.model.draft3.graph.expression.{FileEvaluator, TypeEvaluator, ValueEvaluator} +import wdl.transforms.base.wdlom2wom.TaskDefinitionElementToWomTaskDefinition.{ + createTaskGraph, + eliminateInputDependencies, + expandLines, + processMetaSections, + validateParameterMetaEntries, + TaskDefinitionElementToWomInputs +} import wdl.transforms.base.wdlom2wom.WorkflowDefinitionElementToWomWorkflowDefinition.WorkflowDefinitionConvertInputs import wdl.transforms.base.wdlom2wom.{ + CommandPartElementToWomCommandPart, FileElementToWomBundle, FileElementToWomBundleInputs, TaskDefinitionElementToWomTaskDefinition, @@ -15,13 +29,118 @@ import wdl.transforms.cascades.linking.expression.consumed._ import wdl.transforms.cascades.linking.expression.files._ import wdl.transforms.cascades.linking.expression.types._ import wdl.transforms.cascades.linking.expression.values._ +import wom.values.WomInteger +import wom.{CommandPart, RuntimeAttributes, RuntimeAttributesKeys} package object wdlom2wom { val taskDefinitionElementToWomTaskDefinition: CheckedAtoB[TaskDefinitionElementToWomInputs, CallableTaskDefinition] = - CheckedAtoB.fromErrorOr(TaskDefinitionElementToWomTaskDefinition.convert) + CheckedAtoB.fromErrorOr(convert) val workflowDefinitionElementToWomWorkflowDefinition : CheckedAtoB[WorkflowDefinitionConvertInputs, WorkflowDefinition] = CheckedAtoB.fromErrorOr(WorkflowDefinitionElementToWomWorkflowDefinition.convert) val fileElementToWomBundle: CheckedAtoB[FileElementToWomBundleInputs, WomBundle] = CheckedAtoB.fromCheck(FileElementToWomBundle.convert) + + private def convert(b: TaskDefinitionElementToWomInputs)(implicit + expressionValueConsumer: ExpressionValueConsumer[ExpressionElement], + fileEvaluator: FileEvaluator[ExpressionElement], + typeEvaluator: TypeEvaluator[ExpressionElement], + valueEvaluator: ValueEvaluator[ExpressionElement] + ): ErrorOr[CallableTaskDefinition] = { + val a = eliminateInputDependencies(b)(expressionValueConsumer) + val inputElements = a.taskDefinitionElement.inputsSection.map(_.inputDeclarations).getOrElse(Seq.empty) + + val declarations = a.taskDefinitionElement.declarations + val outputElements = a.taskDefinitionElement.outputsSection.map(_.outputs).getOrElse(Seq.empty) + + val conversion = ( + createTaskGraph(inputElements, + declarations, + outputElements, + a.taskDefinitionElement.parameterMetaSection, + a.typeAliases + )(expressionValueConsumer, fileEvaluator, typeEvaluator, valueEvaluator), + validateParameterMetaEntries(a.taskDefinitionElement.parameterMetaSection, + a.taskDefinitionElement.inputsSection, + a.taskDefinitionElement.outputsSection + ) + ) flatMapN { (taskGraph, _) => + val validRuntimeAttributes: ErrorOr[RuntimeAttributes] = a.taskDefinitionElement.runtimeSection match { + case Some(attributeSection) => + val returnCodesAttribute = + attributeSection.runtimeAttributes.toList.find(pair => + pair.key.equals(RuntimeAttributesKeys.ReturnCodesKey) + ) + val continueOnReturnCodeAttribute = + attributeSection.runtimeAttributes.toList.find(pair => + pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) + ) + + val returnCodesGet = returnCodesAttribute.orNull + val continueOnReturnCodeGet = continueOnReturnCodeAttribute.orNull + var editedAttributes = attributeSection.runtimeAttributes + var returnCodesNotUnique = false + + if (returnCodesGet != null && continueOnReturnCodeGet != null) { + returnCodesNotUnique = returnCodesGet.value + .equals(continueOnReturnCodeGet.value) || returnCodesGet.value.equals( + ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) + ) + } + + if (returnCodesGet != null && !returnCodesNotUnique) { + editedAttributes = attributeSection.runtimeAttributes.filterNot(attribute => + attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) + ) + editedAttributes = editedAttributes ++ Vector( + KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesAttribute.get.value) + ) + } + + editedAttributes = + editedAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) + + TaskDefinitionElementToWomTaskDefinition.createRuntimeAttributes( + RuntimeAttributesSectionElement(editedAttributes), + taskGraph.linkedGraph + )( + expressionValueConsumer, + fileEvaluator, + typeEvaluator, + valueEvaluator + ) + case None => RuntimeAttributes(Map.empty).validNel + } + + val validCommand: ErrorOr[Seq[CommandPart]] = + expandLines(a.taskDefinitionElement.commandSection.parts).toList + .traverse { parts => + CommandPartElementToWomCommandPart.convert(parts, + taskGraph.linkedGraph.typeAliases, + taskGraph.linkedGraph.generatedHandles + )(expressionValueConsumer, fileEvaluator, typeEvaluator, valueEvaluator) + } + .map(_.toSeq) + + val (meta, parameterMeta) = + processMetaSections(a.taskDefinitionElement.metaSection, a.taskDefinitionElement.parameterMetaSection) + + (validRuntimeAttributes, validCommand) mapN { (runtime, command) => + CallableTaskDefinition( + a.taskDefinitionElement.name, + Function.const(command.validNel), + runtime, + meta, + parameterMeta, + taskGraph.outputs, + taskGraph.inputs, + Set.empty, + Map.empty, + sourceLocation = a.taskDefinitionElement.sourceLocation + ) + } + } + + conversion.contextualizeErrors(s"process task definition '${b.taskDefinitionElement.name}'") + } } diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala index 267b38fcfc8..e001219c659 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala @@ -100,9 +100,9 @@ object TaskDefinitionElementToWomTaskDefinition extends Util { conversion.contextualizeErrors(s"process task definition '${b.taskDefinitionElement.name}'") } - private def validateParameterMetaEntries(parameterMetaSectionElement: Option[ParameterMetaSectionElement], - inputs: Option[InputsSectionElement], - outputs: Option[OutputsSectionElement] + def validateParameterMetaEntries(parameterMetaSectionElement: Option[ParameterMetaSectionElement], + inputs: Option[InputsSectionElement], + outputs: Option[OutputsSectionElement] ): ErrorOr[Unit] = { val validKeys: List[String] = inputs.toList.flatMap(_.inputDeclarations.map(_.name)) ++ outputs.toList.flatMap(_.outputs.map(_.name)) @@ -123,7 +123,7 @@ object TaskDefinitionElementToWomTaskDefinition extends Util { } } - private def eliminateInputDependencies( + def eliminateInputDependencies( a: TaskDefinitionElementToWomInputs )(implicit expressionValueConsumer: ExpressionValueConsumer[ExpressionElement]): TaskDefinitionElementToWomInputs = { case class NewInputElementsSet(original: InputDeclarationElement, @@ -208,7 +208,7 @@ object TaskDefinitionElementToWomTaskDefinition extends Util { ) } - private def expandLines(lines: Seq[CommandSectionLine]): Seq[CommandPartElement] = { + def expandLines(lines: Seq[CommandSectionLine]): Seq[CommandPartElement] = { def expandNonFinalLine(line: CommandSectionLine): Seq[CommandPartElement] = if (line.parts.isEmpty) { Seq(StringCommandPartElement(System.lineSeparator)) } else { @@ -228,16 +228,16 @@ object TaskDefinitionElementToWomTaskDefinition extends Util { } } - final private case class TaskGraph(inputs: List[Callable.InputDefinition], - outputs: List[Callable.OutputDefinition], - linkedGraph: LinkedGraph + final case class TaskGraph(inputs: List[Callable.InputDefinition], + outputs: List[Callable.OutputDefinition], + linkedGraph: LinkedGraph ) - private def createTaskGraph(inputs: Seq[InputDeclarationElement], - declarations: Seq[IntermediateValueDeclarationElement], - outputs: Seq[OutputDeclarationElement], - parameterMeta: Option[ParameterMetaSectionElement], - typeAliases: Map[String, WomType] + def createTaskGraph(inputs: Seq[InputDeclarationElement], + declarations: Seq[IntermediateValueDeclarationElement], + outputs: Seq[OutputDeclarationElement], + parameterMeta: Option[ParameterMetaSectionElement], + typeAliases: Map[String, WomType] )(implicit expressionValueConsumer: ExpressionValueConsumer[ExpressionElement], fileEvaluator: FileEvaluator[ExpressionElement], @@ -326,7 +326,7 @@ object TaskDefinitionElementToWomTaskDefinition extends Util { } } - private def createRuntimeAttributes(attributes: RuntimeAttributesSectionElement, linkedGraph: LinkedGraph)(implicit + def createRuntimeAttributes(attributes: RuntimeAttributesSectionElement, linkedGraph: LinkedGraph)(implicit expressionValueConsumer: ExpressionValueConsumer[ExpressionElement], fileEvaluator: FileEvaluator[ExpressionElement], typeEvaluator: TypeEvaluator[ExpressionElement], @@ -340,36 +340,7 @@ object TaskDefinitionElementToWomTaskDefinition extends Util { womExpression <- kvPair.value.makeWomExpression(linkedGraph.typeAliases, consumedValueLookup) } yield kvPair.key -> womExpression - val returnCodesAttribute = - attributes.runtimeAttributes.toList.find(pair => pair.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) - val continueOnReturnCodeAttribute = - attributes.runtimeAttributes.toList.find(pair => pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey)) - - val returnCodesGet = returnCodesAttribute.orNull - val continueOnReturnCodeGet = continueOnReturnCodeAttribute.orNull - var editedAttributes = attributes.runtimeAttributes - - if (returnCodesGet != null) { - val returnCodesNotUnique = - returnCodesGet.value - .equals(continueOnReturnCodeGet.value) || returnCodesGet.value.equals( - ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) - ) - - if (!returnCodesNotUnique) { - editedAttributes = attributes.runtimeAttributes.filterNot(attribute => - attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) - ) - editedAttributes = editedAttributes ++ Vector( - KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesAttribute.get.value) - ) - } - } - - editedAttributes = - editedAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) - - editedAttributes.toList + attributes.runtimeAttributes.toList .traverse(processSingleRuntimeAttribute) .map(atts => RuntimeAttributes(atts.toMap)) } From 58eb36d530ef46aae37be5bbf6ea238d020a4e78 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Mon, 8 Apr 2024 13:42:05 -0400 Subject: [PATCH 19/23] Optimize imports --- ...DefinitionElementToWomTaskDefinition.scala | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala index e001219c659..7eeb54b8cab 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala @@ -2,36 +2,29 @@ package wdl.transforms.base.wdlom2wom import cats.data.NonEmptyList import cats.data.Validated.{Invalid, Valid} +import cats.instances.list._ import cats.syntax.apply._ import cats.syntax.traverse._ import cats.syntax.validated._ -import cats.instances.list._ import common.validation.ErrorOr.{ErrorOr, _} import wdl.model.draft3.elements.CommandPartElement.{PlaceholderCommandPartElement, StringCommandPartElement} -import wdl.model.draft3.elements.ExpressionElement.{ - ArrayLiteral, - IdentifierLookup, - KvPair, - PrimitiveLiteralExpressionElement, - SelectFirst -} +import wdl.model.draft3.elements.ExpressionElement.{ArrayLiteral, IdentifierLookup, KvPair, SelectFirst} import wdl.model.draft3.elements._ -import wdl.model.draft3.graph.{ExpressionValueConsumer, LinkedGraph} -import wom.callable.Callable._ -import wom.callable.{Callable, CallableTaskDefinition, MetaValueElement} -import wom.expression.WomExpression -import wom.types.{WomOptionalType, WomType} -import wom.{CommandPart, RuntimeAttributes, RuntimeAttributesKeys} import wdl.model.draft3.graph.ExpressionValueConsumer.ops._ -import wdl.model.draft3.graph.expression.{FileEvaluator, TypeEvaluator, ValueEvaluator} import wdl.model.draft3.graph.expression.WomExpressionMaker.ops._ +import wdl.model.draft3.graph.expression.WomTypeMaker.ops._ +import wdl.model.draft3.graph.expression.{FileEvaluator, TypeEvaluator, ValueEvaluator} +import wdl.model.draft3.graph.{ExpressionValueConsumer, LinkedGraph} import wdl.transforms.base.linking.expression._ import wdl.transforms.base.linking.graph.LinkedGraphMaker +import wdl.transforms.base.linking.typemakers._ import wdl.transforms.base.wdlom2wom.expression.renaming.IdentifierLookupRenamer.ops._ import wdl.transforms.base.wdlom2wom.expression.renaming.expressionEvaluator -import wdl.model.draft3.graph.expression.WomTypeMaker.ops._ -import wdl.transforms.base.linking.typemakers._ -import wom.values.WomInteger +import wom.callable.Callable._ +import wom.callable.{Callable, CallableTaskDefinition, MetaValueElement} +import wom.expression.WomExpression +import wom.types.{WomOptionalType, WomType} +import wom.{CommandPart, RuntimeAttributes} object TaskDefinitionElementToWomTaskDefinition extends Util { From 9876bb410f815c1821d8668c3b04d82bad40d359 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Wed, 10 Apr 2024 11:35:35 -0400 Subject: [PATCH 20/23] Refactored to reduce code duplication --- .../biscayne/wdlom2wom/package.scala | 147 +++++++----------- .../cascades/wdlom2wom/package.scala | 147 +++++++----------- ...DefinitionElementToWomTaskDefinition.scala | 125 ++++++++------- 3 files changed, 184 insertions(+), 235 deletions(-) diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala index 5fb11a1de10..ab62513db01 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala @@ -1,6 +1,6 @@ package wdl.transforms.biscayne -import cats.implicits.{catsSyntaxTuple2Semigroupal, catsSyntaxValidatedId, toTraverseOps} +import cats.implicits.catsSyntaxValidatedId import common.transforms.CheckedAtoB import common.validation.ErrorOr.{ErrorOr, ShortCircuitingFlatMapTuple2, _} import wdl.model.draft3.elements.{ExpressionElement, RuntimeAttributesSectionElement} @@ -8,16 +8,11 @@ import wdl.model.draft3.elements.ExpressionElement.{ArrayLiteral, KvPair, Primit import wdl.model.draft3.graph.ExpressionValueConsumer import wdl.model.draft3.graph.expression.{FileEvaluator, TypeEvaluator, ValueEvaluator} import wdl.transforms.base.wdlom2wom.TaskDefinitionElementToWomTaskDefinition.{ - createTaskGraph, eliminateInputDependencies, - expandLines, - processMetaSections, - validateParameterMetaEntries, TaskDefinitionElementToWomInputs } import wdl.transforms.base.wdlom2wom.WorkflowDefinitionElementToWomWorkflowDefinition.WorkflowDefinitionConvertInputs import wdl.transforms.base.wdlom2wom.{ - CommandPartElementToWomCommandPart, FileElementToWomBundle, FileElementToWomBundleInputs, TaskDefinitionElementToWomTaskDefinition, @@ -30,7 +25,7 @@ import wdl.transforms.biscayne.linking.expression.files._ import wdl.transforms.biscayne.linking.expression.types._ import wdl.transforms.biscayne.linking.expression.values._ import wom.values.WomInteger -import wom.{CommandPart, RuntimeAttributes, RuntimeAttributesKeys} +import wom.{RuntimeAttributes, RuntimeAttributesKeys} package object wdlom2wom { val taskDefinitionElementToWomTaskDefinition: CheckedAtoB[TaskDefinitionElementToWomInputs, CallableTaskDefinition] = @@ -48,99 +43,69 @@ package object wdlom2wom { valueEvaluator: ValueEvaluator[ExpressionElement] ): ErrorOr[CallableTaskDefinition] = { val a = eliminateInputDependencies(b)(expressionValueConsumer) - val inputElements = a.taskDefinitionElement.inputsSection.map(_.inputDeclarations).getOrElse(Seq.empty) - val declarations = a.taskDefinitionElement.declarations - val outputElements = a.taskDefinitionElement.outputsSection.map(_.outputs).getOrElse(Seq.empty) - - val conversion = ( - createTaskGraph(inputElements, - declarations, - outputElements, - a.taskDefinitionElement.parameterMetaSection, - a.typeAliases - )(expressionValueConsumer, fileEvaluator, typeEvaluator, valueEvaluator), - validateParameterMetaEntries(a.taskDefinitionElement.parameterMetaSection, - a.taskDefinitionElement.inputsSection, - a.taskDefinitionElement.outputsSection - ) - ) flatMapN { (taskGraph, _) => - val validRuntimeAttributes: ErrorOr[RuntimeAttributes] = a.taskDefinitionElement.runtimeSection match { - case Some(attributeSection) => - val returnCodesAttribute = - attributeSection.runtimeAttributes.toList.find(pair => - pair.key.equals(RuntimeAttributesKeys.ReturnCodesKey) - ) - val continueOnReturnCodeAttribute = - attributeSection.runtimeAttributes.toList.find(pair => - pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) - ) - - val returnCodesGet = returnCodesAttribute.orNull - val continueOnReturnCodeGet = continueOnReturnCodeAttribute.orNull - var editedAttributes = attributeSection.runtimeAttributes - var returnCodesNotUnique = false - - if (returnCodesGet != null && continueOnReturnCodeGet != null) { - returnCodesNotUnique = returnCodesGet.value - .equals(continueOnReturnCodeGet.value) || returnCodesGet.value.equals( - ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) + val conversion = + TaskDefinitionElementToWomTaskDefinition.createTaskGraphAndValidateMetadata(a)(expressionValueConsumer, + fileEvaluator, + typeEvaluator, + valueEvaluator + ) flatMapN { (taskGraph, _) => + val validRuntimeAttributes: ErrorOr[RuntimeAttributes] = a.taskDefinitionElement.runtimeSection match { + case Some(attributeSection) => + TaskDefinitionElementToWomTaskDefinition.createRuntimeAttributes( + RuntimeAttributesSectionElement(getFinalRuntimeAttributes(attributeSection)), + taskGraph.linkedGraph + )( + expressionValueConsumer, + fileEvaluator, + typeEvaluator, + valueEvaluator ) - } + case None => RuntimeAttributes(Map.empty).validNel + } - if (returnCodesGet != null && !returnCodesNotUnique) { - editedAttributes = attributeSection.runtimeAttributes.filterNot(attribute => - attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) - ) - editedAttributes = editedAttributes ++ Vector( - KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesAttribute.get.value) - ) - } + TaskDefinitionElementToWomTaskDefinition.createCallableTaskDefinition(a, taskGraph, validRuntimeAttributes)( + expressionValueConsumer, + fileEvaluator, + typeEvaluator, + valueEvaluator + ) + } - editedAttributes = - editedAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) + conversion.contextualizeErrors(s"process task definition '${b.taskDefinitionElement.name}'") + } - TaskDefinitionElementToWomTaskDefinition.createRuntimeAttributes( - RuntimeAttributesSectionElement(editedAttributes), - taskGraph.linkedGraph - )( - expressionValueConsumer, - fileEvaluator, - typeEvaluator, - valueEvaluator - ) - case None => RuntimeAttributes(Map.empty).validNel - } + private def getFinalRuntimeAttributes(attributeSection: RuntimeAttributesSectionElement): Vector[KvPair] = { + val returnCodesAttribute = + attributeSection.runtimeAttributes.toList.find(pair => pair.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) + val continueOnReturnCodeAttribute = + attributeSection.runtimeAttributes.toList.find(pair => + pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) + ) - val validCommand: ErrorOr[Seq[CommandPart]] = - expandLines(a.taskDefinitionElement.commandSection.parts).toList - .traverse { parts => - CommandPartElementToWomCommandPart.convert(parts, - taskGraph.linkedGraph.typeAliases, - taskGraph.linkedGraph.generatedHandles - )(expressionValueConsumer, fileEvaluator, typeEvaluator, valueEvaluator) - } - .map(_.toSeq) + val returnCodesGet = returnCodesAttribute.orNull + val continueOnReturnCodeGet = continueOnReturnCodeAttribute.orNull + var editedAttributes = attributeSection.runtimeAttributes + var returnCodesNotUnique = false - val (meta, parameterMeta) = - processMetaSections(a.taskDefinitionElement.metaSection, a.taskDefinitionElement.parameterMetaSection) + if (returnCodesGet != null && continueOnReturnCodeGet != null) { + returnCodesNotUnique = returnCodesGet.value + .equals(continueOnReturnCodeGet.value) || returnCodesGet.value.equals( + ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) + ) + } - (validRuntimeAttributes, validCommand) mapN { (runtime, command) => - CallableTaskDefinition( - a.taskDefinitionElement.name, - Function.const(command.validNel), - runtime, - meta, - parameterMeta, - taskGraph.outputs, - taskGraph.inputs, - Set.empty, - Map.empty, - sourceLocation = a.taskDefinitionElement.sourceLocation - ) - } + if (returnCodesGet != null && !returnCodesNotUnique) { + editedAttributes = attributeSection.runtimeAttributes.filterNot(attribute => + attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) + ) + editedAttributes = editedAttributes ++ Vector( + KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesAttribute.get.value) + ) } - conversion.contextualizeErrors(s"process task definition '${b.taskDefinitionElement.name}'") + editedAttributes = + editedAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) + editedAttributes } } diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala index 35f2fa8b8cc..7ec56a28402 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala @@ -1,6 +1,6 @@ package wdl.transforms.cascades -import cats.implicits.{catsSyntaxTuple2Semigroupal, catsSyntaxValidatedId, toTraverseOps} +import cats.implicits.catsSyntaxValidatedId import common.transforms.CheckedAtoB import common.validation.ErrorOr.{ErrorOr, ShortCircuitingFlatMapTuple2, _} import wdl.model.draft3.elements.ExpressionElement.{ArrayLiteral, KvPair, PrimitiveLiteralExpressionElement} @@ -8,16 +8,11 @@ import wdl.model.draft3.elements.{ExpressionElement, RuntimeAttributesSectionEle import wdl.model.draft3.graph.ExpressionValueConsumer import wdl.model.draft3.graph.expression.{FileEvaluator, TypeEvaluator, ValueEvaluator} import wdl.transforms.base.wdlom2wom.TaskDefinitionElementToWomTaskDefinition.{ - createTaskGraph, eliminateInputDependencies, - expandLines, - processMetaSections, - validateParameterMetaEntries, TaskDefinitionElementToWomInputs } import wdl.transforms.base.wdlom2wom.WorkflowDefinitionElementToWomWorkflowDefinition.WorkflowDefinitionConvertInputs import wdl.transforms.base.wdlom2wom.{ - CommandPartElementToWomCommandPart, FileElementToWomBundle, FileElementToWomBundleInputs, TaskDefinitionElementToWomTaskDefinition, @@ -30,7 +25,7 @@ import wdl.transforms.cascades.linking.expression.files._ import wdl.transforms.cascades.linking.expression.types._ import wdl.transforms.cascades.linking.expression.values._ import wom.values.WomInteger -import wom.{CommandPart, RuntimeAttributes, RuntimeAttributesKeys} +import wom.{RuntimeAttributes, RuntimeAttributesKeys} package object wdlom2wom { val taskDefinitionElementToWomTaskDefinition: CheckedAtoB[TaskDefinitionElementToWomInputs, CallableTaskDefinition] = @@ -48,99 +43,69 @@ package object wdlom2wom { valueEvaluator: ValueEvaluator[ExpressionElement] ): ErrorOr[CallableTaskDefinition] = { val a = eliminateInputDependencies(b)(expressionValueConsumer) - val inputElements = a.taskDefinitionElement.inputsSection.map(_.inputDeclarations).getOrElse(Seq.empty) - val declarations = a.taskDefinitionElement.declarations - val outputElements = a.taskDefinitionElement.outputsSection.map(_.outputs).getOrElse(Seq.empty) - - val conversion = ( - createTaskGraph(inputElements, - declarations, - outputElements, - a.taskDefinitionElement.parameterMetaSection, - a.typeAliases - )(expressionValueConsumer, fileEvaluator, typeEvaluator, valueEvaluator), - validateParameterMetaEntries(a.taskDefinitionElement.parameterMetaSection, - a.taskDefinitionElement.inputsSection, - a.taskDefinitionElement.outputsSection - ) - ) flatMapN { (taskGraph, _) => - val validRuntimeAttributes: ErrorOr[RuntimeAttributes] = a.taskDefinitionElement.runtimeSection match { - case Some(attributeSection) => - val returnCodesAttribute = - attributeSection.runtimeAttributes.toList.find(pair => - pair.key.equals(RuntimeAttributesKeys.ReturnCodesKey) - ) - val continueOnReturnCodeAttribute = - attributeSection.runtimeAttributes.toList.find(pair => - pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) - ) - - val returnCodesGet = returnCodesAttribute.orNull - val continueOnReturnCodeGet = continueOnReturnCodeAttribute.orNull - var editedAttributes = attributeSection.runtimeAttributes - var returnCodesNotUnique = false - - if (returnCodesGet != null && continueOnReturnCodeGet != null) { - returnCodesNotUnique = returnCodesGet.value - .equals(continueOnReturnCodeGet.value) || returnCodesGet.value.equals( - ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) + val conversion = + TaskDefinitionElementToWomTaskDefinition.createTaskGraphAndValidateMetadata(a)(expressionValueConsumer, + fileEvaluator, + typeEvaluator, + valueEvaluator + ) flatMapN { (taskGraph, _) => + val validRuntimeAttributes: ErrorOr[RuntimeAttributes] = a.taskDefinitionElement.runtimeSection match { + case Some(attributeSection) => + TaskDefinitionElementToWomTaskDefinition.createRuntimeAttributes( + RuntimeAttributesSectionElement(getFinalRuntimeAttributes(attributeSection)), + taskGraph.linkedGraph + )( + expressionValueConsumer, + fileEvaluator, + typeEvaluator, + valueEvaluator ) - } + case None => RuntimeAttributes(Map.empty).validNel + } - if (returnCodesGet != null && !returnCodesNotUnique) { - editedAttributes = attributeSection.runtimeAttributes.filterNot(attribute => - attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) - ) - editedAttributes = editedAttributes ++ Vector( - KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesAttribute.get.value) - ) - } + TaskDefinitionElementToWomTaskDefinition.createCallableTaskDefinition(a, taskGraph, validRuntimeAttributes)( + expressionValueConsumer, + fileEvaluator, + typeEvaluator, + valueEvaluator + ) + } - editedAttributes = - editedAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) + conversion.contextualizeErrors(s"process task definition '${b.taskDefinitionElement.name}'") + } - TaskDefinitionElementToWomTaskDefinition.createRuntimeAttributes( - RuntimeAttributesSectionElement(editedAttributes), - taskGraph.linkedGraph - )( - expressionValueConsumer, - fileEvaluator, - typeEvaluator, - valueEvaluator - ) - case None => RuntimeAttributes(Map.empty).validNel - } + private def getFinalRuntimeAttributes(attributeSection: RuntimeAttributesSectionElement): Vector[KvPair] = { + val returnCodesAttribute = + attributeSection.runtimeAttributes.toList.find(pair => pair.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) + val continueOnReturnCodeAttribute = + attributeSection.runtimeAttributes.toList.find(pair => + pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) + ) - val validCommand: ErrorOr[Seq[CommandPart]] = - expandLines(a.taskDefinitionElement.commandSection.parts).toList - .traverse { parts => - CommandPartElementToWomCommandPart.convert(parts, - taskGraph.linkedGraph.typeAliases, - taskGraph.linkedGraph.generatedHandles - )(expressionValueConsumer, fileEvaluator, typeEvaluator, valueEvaluator) - } - .map(_.toSeq) + val returnCodesGet = returnCodesAttribute.orNull + val continueOnReturnCodeGet = continueOnReturnCodeAttribute.orNull + var editedAttributes = attributeSection.runtimeAttributes + var returnCodesNotUnique = false - val (meta, parameterMeta) = - processMetaSections(a.taskDefinitionElement.metaSection, a.taskDefinitionElement.parameterMetaSection) + if (returnCodesGet != null && continueOnReturnCodeGet != null) { + returnCodesNotUnique = returnCodesGet.value + .equals(continueOnReturnCodeGet.value) || returnCodesGet.value.equals( + ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) + ) + } - (validRuntimeAttributes, validCommand) mapN { (runtime, command) => - CallableTaskDefinition( - a.taskDefinitionElement.name, - Function.const(command.validNel), - runtime, - meta, - parameterMeta, - taskGraph.outputs, - taskGraph.inputs, - Set.empty, - Map.empty, - sourceLocation = a.taskDefinitionElement.sourceLocation - ) - } + if (returnCodesGet != null && !returnCodesNotUnique) { + editedAttributes = attributeSection.runtimeAttributes.filterNot(attribute => + attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) + ) + editedAttributes = editedAttributes ++ Vector( + KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesAttribute.get.value) + ) } - conversion.contextualizeErrors(s"process task definition '${b.taskDefinitionElement.name}'") + editedAttributes = + editedAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) + editedAttributes } } diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala index 7eeb54b8cab..d6e0eba7b42 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/TaskDefinitionElementToWomTaskDefinition.scala @@ -39,63 +39,82 @@ object TaskDefinitionElementToWomTaskDefinition extends Util { valueEvaluator: ValueEvaluator[ExpressionElement] ): ErrorOr[CallableTaskDefinition] = { val a = eliminateInputDependencies(b) - val inputElements = a.taskDefinitionElement.inputsSection.map(_.inputDeclarations).getOrElse(Seq.empty) - - val declarations = a.taskDefinitionElement.declarations - val outputElements = a.taskDefinitionElement.outputsSection.map(_.outputs).getOrElse(Seq.empty) - val conversion = ( - createTaskGraph(inputElements, - declarations, - outputElements, - a.taskDefinitionElement.parameterMetaSection, - a.typeAliases - ), - validateParameterMetaEntries(a.taskDefinitionElement.parameterMetaSection, - a.taskDefinitionElement.inputsSection, - a.taskDefinitionElement.outputsSection - ) - ) flatMapN { (taskGraph, _) => + val conversion = createTaskGraphAndValidateMetadata(a) flatMapN { (taskGraph, _) => val validRuntimeAttributes: ErrorOr[RuntimeAttributes] = a.taskDefinitionElement.runtimeSection match { case Some(attributeSection) => createRuntimeAttributes(attributeSection, taskGraph.linkedGraph) case None => RuntimeAttributes(Map.empty).validNel } - - val validCommand: ErrorOr[Seq[CommandPart]] = - expandLines(a.taskDefinitionElement.commandSection.parts).toList - .traverse { parts => - CommandPartElementToWomCommandPart.convert(parts, - taskGraph.linkedGraph.typeAliases, - taskGraph.linkedGraph.generatedHandles - ) - } - .map(_.toSeq) - - val (meta, parameterMeta) = - processMetaSections(a.taskDefinitionElement.metaSection, a.taskDefinitionElement.parameterMetaSection) - - (validRuntimeAttributes, validCommand) mapN { (runtime, command) => - CallableTaskDefinition( - a.taskDefinitionElement.name, - Function.const(command.validNel), - runtime, - meta, - parameterMeta, - taskGraph.outputs, - taskGraph.inputs, - Set.empty, - Map.empty, - sourceLocation = a.taskDefinitionElement.sourceLocation - ) - } + createCallableTaskDefinition(a, taskGraph, validRuntimeAttributes) } conversion.contextualizeErrors(s"process task definition '${b.taskDefinitionElement.name}'") } - def validateParameterMetaEntries(parameterMetaSectionElement: Option[ParameterMetaSectionElement], - inputs: Option[InputsSectionElement], - outputs: Option[OutputsSectionElement] + def createTaskGraphAndValidateMetadata(a: TaskDefinitionElementToWomInputs)(implicit + expressionValueConsumer: ExpressionValueConsumer[ExpressionElement], + fileEvaluator: FileEvaluator[ExpressionElement], + typeEvaluator: TypeEvaluator[ExpressionElement], + valueEvaluator: ValueEvaluator[ExpressionElement] + ): (ErrorOr[TaskGraph], ErrorOr[Unit]) = { + val inputElements = a.taskDefinitionElement.inputsSection.map(_.inputDeclarations).getOrElse(Seq.empty) + + val declarations = a.taskDefinitionElement.declarations + val outputElements = a.taskDefinitionElement.outputsSection.map(_.outputs).getOrElse(Seq.empty) + + (createTaskGraph(inputElements, + declarations, + outputElements, + a.taskDefinitionElement.parameterMetaSection, + a.typeAliases + ), + validateParameterMetaEntries(a.taskDefinitionElement.parameterMetaSection, + a.taskDefinitionElement.inputsSection, + a.taskDefinitionElement.outputsSection + ) + ) + } + + def createCallableTaskDefinition(a: TaskDefinitionElementToWomInputs, + taskGraph: TaskGraph, + validRuntimeAttributes: ErrorOr[RuntimeAttributes] + )(implicit + expressionValueConsumer: ExpressionValueConsumer[ExpressionElement], + fileEvaluator: FileEvaluator[ExpressionElement], + typeEvaluator: TypeEvaluator[ExpressionElement], + valueEvaluator: ValueEvaluator[ExpressionElement] + ): ErrorOr[CallableTaskDefinition] = { + val validCommand: ErrorOr[Seq[CommandPart]] = + expandLines(a.taskDefinitionElement.commandSection.parts).toList + .traverse { parts => + CommandPartElementToWomCommandPart.convert(parts, + taskGraph.linkedGraph.typeAliases, + taskGraph.linkedGraph.generatedHandles + ) + } + .map(_.toSeq) + + val (meta, parameterMeta) = + processMetaSections(a.taskDefinitionElement.metaSection, a.taskDefinitionElement.parameterMetaSection) + + (validRuntimeAttributes, validCommand) mapN { (runtime, command) => + CallableTaskDefinition( + a.taskDefinitionElement.name, + Function.const(command.validNel), + runtime, + meta, + parameterMeta, + taskGraph.outputs, + taskGraph.inputs, + Set.empty, + Map.empty, + sourceLocation = a.taskDefinitionElement.sourceLocation + ) + } + } + private def validateParameterMetaEntries(parameterMetaSectionElement: Option[ParameterMetaSectionElement], + inputs: Option[InputsSectionElement], + outputs: Option[OutputsSectionElement] ): ErrorOr[Unit] = { val validKeys: List[String] = inputs.toList.flatMap(_.inputDeclarations.map(_.name)) ++ outputs.toList.flatMap(_.outputs.map(_.name)) @@ -201,7 +220,7 @@ object TaskDefinitionElementToWomTaskDefinition extends Util { ) } - def expandLines(lines: Seq[CommandSectionLine]): Seq[CommandPartElement] = { + private def expandLines(lines: Seq[CommandSectionLine]): Seq[CommandPartElement] = { def expandNonFinalLine(line: CommandSectionLine): Seq[CommandPartElement] = if (line.parts.isEmpty) { Seq(StringCommandPartElement(System.lineSeparator)) } else { @@ -226,11 +245,11 @@ object TaskDefinitionElementToWomTaskDefinition extends Util { linkedGraph: LinkedGraph ) - def createTaskGraph(inputs: Seq[InputDeclarationElement], - declarations: Seq[IntermediateValueDeclarationElement], - outputs: Seq[OutputDeclarationElement], - parameterMeta: Option[ParameterMetaSectionElement], - typeAliases: Map[String, WomType] + private def createTaskGraph(inputs: Seq[InputDeclarationElement], + declarations: Seq[IntermediateValueDeclarationElement], + outputs: Seq[OutputDeclarationElement], + parameterMeta: Option[ParameterMetaSectionElement], + typeAliases: Map[String, WomType] )(implicit expressionValueConsumer: ExpressionValueConsumer[ExpressionElement], fileEvaluator: FileEvaluator[ExpressionElement], From 27079d7ecdef2a3b65dd652a695cf80cbf4a23c6 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Wed, 10 Apr 2024 16:33:14 -0400 Subject: [PATCH 21/23] Added comments and 'scala-ified' code --- .../biscayne/wdlom2wom/package.scala | 39 ++++++++++++------- .../cascades/wdlom2wom/package.scala | 39 ++++++++++++------- .../main/scala/wom/RuntimeAttributes.scala | 1 + 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala index ab62513db01..faa72151ebe 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala @@ -75,6 +75,15 @@ package object wdlom2wom { conversion.contextualizeErrors(s"process task definition '${b.taskDefinitionElement.name}'") } + /** + * Combine `returnCodes` and `continueOnReturnCode` to be a single attribute. The resulting vector will contain + * `continueOnReturnCode` if either `continueOnReturnCode` or `returnCodes` was in the `attributeSection`, it will + * never contain `returnCodes`. The value for the `continueOnReturnCode` key in the new vector will be the value + * associated with `returnCodes` in the original vector if it exists, else it will be the value associated with + * `continueOnReturnCode` in the original vector. + * @param attributeSection list of all runtime attributes and their values + * @return A vector of pairs of runtime attribute keys to their respective values + */ private def getFinalRuntimeAttributes(attributeSection: RuntimeAttributesSectionElement): Vector[KvPair] = { val returnCodesAttribute = attributeSection.runtimeAttributes.toList.find(pair => pair.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) @@ -83,25 +92,25 @@ package object wdlom2wom { pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) ) - val returnCodesGet = returnCodesAttribute.orNull - val continueOnReturnCodeGet = continueOnReturnCodeAttribute.orNull var editedAttributes = attributeSection.runtimeAttributes - var returnCodesNotUnique = false - if (returnCodesGet != null && continueOnReturnCodeGet != null) { - returnCodesNotUnique = returnCodesGet.value - .equals(continueOnReturnCodeGet.value) || returnCodesGet.value.equals( - ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) - ) + val returnCodesNotUnique = (returnCodesAttribute, continueOnReturnCodeAttribute) match { + case (Some(returnCodesValue), Some(continueOnReturnCodeValue)) => + returnCodesValue.value + .equals(continueOnReturnCodeValue.value) || returnCodesValue.value.equals( + ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) + ) + case _ => false } - if (returnCodesGet != null && !returnCodesNotUnique) { - editedAttributes = attributeSection.runtimeAttributes.filterNot(attribute => - attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) - ) - editedAttributes = editedAttributes ++ Vector( - KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesAttribute.get.value) - ) + (returnCodesAttribute, returnCodesNotUnique) match { + case (Some(returnCodesValue), false) => + editedAttributes = attributeSection.runtimeAttributes.filterNot(attribute => + attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) + ) + editedAttributes = editedAttributes ++ Vector( + KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesValue.value) + ) } editedAttributes = diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala index 7ec56a28402..0c7ae31c2be 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala @@ -75,6 +75,15 @@ package object wdlom2wom { conversion.contextualizeErrors(s"process task definition '${b.taskDefinitionElement.name}'") } + /** + * Combine `returnCodes` and `continueOnReturnCode` to be a single attribute. The resulting vector will contain + * `continueOnReturnCode` if either `continueOnReturnCode` or `returnCodes` was in the `attributeSection`, it will + * never contain `returnCodes`. The value for the `continueOnReturnCode` key in the new vector will be the value + * associated with `returnCodes` in the original vector if it exists, else it will be the value associated with + * `continueOnReturnCode` in the original vector. + * @param attributeSection list of all runtime attributes and their values + * @return A vector of pairs of runtime attribute keys to their respective values + */ private def getFinalRuntimeAttributes(attributeSection: RuntimeAttributesSectionElement): Vector[KvPair] = { val returnCodesAttribute = attributeSection.runtimeAttributes.toList.find(pair => pair.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) @@ -83,25 +92,25 @@ package object wdlom2wom { pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) ) - val returnCodesGet = returnCodesAttribute.orNull - val continueOnReturnCodeGet = continueOnReturnCodeAttribute.orNull var editedAttributes = attributeSection.runtimeAttributes - var returnCodesNotUnique = false - if (returnCodesGet != null && continueOnReturnCodeGet != null) { - returnCodesNotUnique = returnCodesGet.value - .equals(continueOnReturnCodeGet.value) || returnCodesGet.value.equals( - ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) - ) + val returnCodesNotUnique = (returnCodesAttribute, continueOnReturnCodeAttribute) match { + case (Some(returnCodesValue), Some(continueOnReturnCodeValue)) => + returnCodesValue.value + .equals(continueOnReturnCodeValue.value) || returnCodesValue.value.equals( + ArrayLiteral(Vector(PrimitiveLiteralExpressionElement(WomInteger(0)))) + ) + case _ => false } - if (returnCodesGet != null && !returnCodesNotUnique) { - editedAttributes = attributeSection.runtimeAttributes.filterNot(attribute => - attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) - ) - editedAttributes = editedAttributes ++ Vector( - KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesAttribute.get.value) - ) + (returnCodesAttribute, returnCodesNotUnique) match { + case (Some(returnCodesValue), false) => + editedAttributes = attributeSection.runtimeAttributes.filterNot(attribute => + attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) + ) + editedAttributes = editedAttributes ++ Vector( + KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesValue.value) + ) } editedAttributes = diff --git a/wom/src/main/scala/wom/RuntimeAttributes.scala b/wom/src/main/scala/wom/RuntimeAttributes.scala index 4de582f5e1b..216d6404dc5 100644 --- a/wom/src/main/scala/wom/RuntimeAttributes.scala +++ b/wom/src/main/scala/wom/RuntimeAttributes.scala @@ -17,6 +17,7 @@ object RuntimeAttributesKeys { val ContinueOnReturnCodeKey = "continueOnReturnCode" // New for WDL 1.1 + // Semantically, this is the same as continueOnReturnCode as the two attributes are combined at the parsing stage val ReturnCodesKey = "returnCodes" } From 10910673f78cfde0324fbb598285d85da1231700 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Wed, 10 Apr 2024 16:43:18 -0400 Subject: [PATCH 22/23] Fixed failing tests --- .../transforms/biscayne/wdlom2wom/package.scala | 14 +++++--------- .../transforms/cascades/wdlom2wom/package.scala | 14 +++++--------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala index faa72151ebe..0169ab5af62 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/wdlom2wom/package.scala @@ -92,8 +92,6 @@ package object wdlom2wom { pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) ) - var editedAttributes = attributeSection.runtimeAttributes - val returnCodesNotUnique = (returnCodesAttribute, continueOnReturnCodeAttribute) match { case (Some(returnCodesValue), Some(continueOnReturnCodeValue)) => returnCodesValue.value @@ -103,18 +101,16 @@ package object wdlom2wom { case _ => false } - (returnCodesAttribute, returnCodesNotUnique) match { + val finalAttributes = (returnCodesAttribute, returnCodesNotUnique) match { case (Some(returnCodesValue), false) => - editedAttributes = attributeSection.runtimeAttributes.filterNot(attribute => + attributeSection.runtimeAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) - ) - editedAttributes = editedAttributes ++ Vector( + ) ++ Vector( KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesValue.value) ) + case _ => attributeSection.runtimeAttributes } - editedAttributes = - editedAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) - editedAttributes + finalAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) } } diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala index 0c7ae31c2be..6441c46f09f 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/wdlom2wom/package.scala @@ -92,8 +92,6 @@ package object wdlom2wom { pair.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) ) - var editedAttributes = attributeSection.runtimeAttributes - val returnCodesNotUnique = (returnCodesAttribute, continueOnReturnCodeAttribute) match { case (Some(returnCodesValue), Some(continueOnReturnCodeValue)) => returnCodesValue.value @@ -103,18 +101,16 @@ package object wdlom2wom { case _ => false } - (returnCodesAttribute, returnCodesNotUnique) match { + val finalAttributes = (returnCodesAttribute, returnCodesNotUnique) match { case (Some(returnCodesValue), false) => - editedAttributes = attributeSection.runtimeAttributes.filterNot(attribute => + attributeSection.runtimeAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ContinueOnReturnCodeKey) - ) - editedAttributes = editedAttributes ++ Vector( + ) ++ Vector( KvPair(RuntimeAttributesKeys.ContinueOnReturnCodeKey, returnCodesValue.value) ) + case _ => attributeSection.runtimeAttributes } - editedAttributes = - editedAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) - editedAttributes + finalAttributes.filterNot(attribute => attribute.key.equals(RuntimeAttributesKeys.ReturnCodesKey)) } } From dde66aaa50974ade6e2956bb060174acf0e829f4 Mon Sep 17 00:00:00 2001 From: Ryan Saperstein Date: Thu, 11 Apr 2024 10:19:29 -0400 Subject: [PATCH 23/23] Added additional tests --- .../validation/ContinueOnReturnCodeSpec.scala | 8 ++++++++ .../backend/validation/ReturnCodesSpec.scala | 19 ------------------- ...turn_codes_invalid_on_old_wdl_version.test | 11 +++++++++++ ...eturn_codes_invalid_on_old_wdl_version.wdl | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+), 19 deletions(-) delete mode 100644 backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala create mode 100644 centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version.test create mode 100644 centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl diff --git a/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala b/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala index e0ff75d3176..bd102bc27c1 100644 --- a/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala @@ -37,5 +37,13 @@ class ContinueOnReturnCodeSpec extends AnyWordSpecLike with CromwellTimeoutSpec ContinueOnReturnCodeSet(set).continueFor(returnCode) should be(expectedContinue) } } + + "continue on expected return code string" in { + val flagTests = Table(("string", "returnCode", "expectedContinue"), ("*", 0, true), ("*", 1, true)) + + forAll(flagTests) { (flag, returnCode, expectedContinue) => + ContinueOnReturnCodeFlag(flag == "*").continueFor(returnCode) should be(expectedContinue) + } + } } } diff --git a/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala b/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala deleted file mode 100644 index 3bd03fc036c..00000000000 --- a/backend/src/test/scala/cromwell/backend/validation/ReturnCodesSpec.scala +++ /dev/null @@ -1,19 +0,0 @@ -package cromwell.backend.validation - -import common.assertion.CromwellTimeoutSpec -import org.scalatest.BeforeAndAfterAll -import org.scalatest.matchers.should.Matchers -import org.scalatest.prop.TableDrivenPropertyChecks._ -import org.scalatest.wordspec.AnyWordSpecLike - -class ReturnCodesSpec extends AnyWordSpecLike with CromwellTimeoutSpec with Matchers with BeforeAndAfterAll { - "Checking for return codes" should { - "continue on expected return code flags" in { - val flagTests = Table(("flag", "returnCode", "expectedContinue"), ("*", 0, true), ("*", 1, true)) - - forAll(flagTests) { (flag, returnCode, expectedContinue) => - ContinueOnReturnCodeFlag(true).continueFor(returnCode) should be(expectedContinue) - } - } - } -} diff --git a/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version.test b/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version.test new file mode 100644 index 00000000000..20c07a6e051 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version.test @@ -0,0 +1,11 @@ +name: return_codes_invalid_on_old_wdl_version +testFormat: workflowfailure + +files { + workflow: return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl +} + +metadata { + workflowName: ReturnCodesInvalidOnOldWdl + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl b/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl new file mode 100644 index 00000000000..296dc833f02 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl @@ -0,0 +1,18 @@ +workflow ReturnCodesInvalidOnOldWdl { + call ReturnCodesInvalidOnOldWdlTask +} + +task ReturnCodesInvalidOnOldWdlTask { + meta { + volatile: "true" + } + + command <<< + exit 5 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [5, 10, 15] + } +}