diff --git a/magenta-lib/src/main/scala/magenta/deployment_type/AutoScaling.scala b/magenta-lib/src/main/scala/magenta/deployment_type/AutoScaling.scala index 54dcb2003..56b110ae9 100644 --- a/magenta-lib/src/main/scala/magenta/deployment_type/AutoScaling.scala +++ b/magenta-lib/src/main/scala/magenta/deployment_type/AutoScaling.scala @@ -16,6 +16,9 @@ import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup import magenta.tasks.{S3 => S3Tasks} import software.amazon.awssdk.services.ssm.SsmClient +import java.time.Duration +import java.time.Duration.{ofMinutes, ofSeconds} + sealed trait MigrationTagRequirements case object NoMigration extends MigrationTagRequirements case object MustBePresent extends MigrationTagRequirements @@ -88,22 +91,24 @@ object AutoScaling extends DeploymentType with BucketParameters { | - scale up, wait for the new instances to become healthy and then scale back down """.stripMargin - val secondsToWait = Param( - "secondsToWait", - "Number of seconds to wait for instances to enter service" - ).default(15 * 60) - val healthcheckGrace = Param( - "healthcheckGrace", - "Number of seconds to wait for the AWS api to stabilise" - ).default(20) - val warmupGrace = Param( - "warmupGrace", - "Number of seconds to wait for the instances in the load balancer to warm up" - ).default(1) - val terminationGrace = Param( - "terminationGrace", - "Number of seconds to wait for the AWS api to stabilise after instance termination" - ).default(10) + val secondsToWait: Param[Duration] = Param + .waitingSecondsFor("secondsToWait", "instances to enter service") + .default(ofMinutes(15)) + val healthcheckGrace: Param[Duration] = Param + .waitingSecondsFor("healthcheckGrace", "the AWS api to stabilise") + .default(ofSeconds(20)) + val warmupGrace: Param[Duration] = Param + .waitingSecondsFor( + "warmupGrace", + "the instances in the load balancer to warm up" + ) + .default(ofSeconds(1)) + val terminationGrace: Param[Duration] = Param + .waitingSecondsFor( + "terminationGrace", + "the AWS api to stabilise after instance termination" + ) + .default(ofSeconds(10)) val prefixStage = Param[Boolean]( "prefixStage", @@ -165,7 +170,7 @@ object AutoScaling extends DeploymentType with BucketParameters { autoScalingGroup: AutoScalingGroupInfo ): List[ASGTask] = { List( - WaitForStabilization(autoScalingGroup, 5 * 60 * 1000, target.region), + WaitForStabilization(autoScalingGroup, ofMinutes(5), target.region), CheckGroupSize(autoScalingGroup, target.region), SuspendAlarmNotifications(autoScalingGroup, target.region), TagCurrentInstancesWithTerminationTag(autoScalingGroup, target.region), @@ -174,32 +179,32 @@ object AutoScaling extends DeploymentType with BucketParameters { HealthcheckGrace( autoScalingGroup, target.region, - healthcheckGrace(pkg, target, reporter) * 1000 + healthcheckGrace(pkg, target, reporter) ), WaitForStabilization( autoScalingGroup, - secondsToWait(pkg, target, reporter) * 1000, + secondsToWait(pkg, target, reporter), target.region ), WarmupGrace( autoScalingGroup, target.region, - warmupGrace(pkg, target, reporter) * 1000 + warmupGrace(pkg, target, reporter) ), WaitForStabilization( autoScalingGroup, - secondsToWait(pkg, target, reporter) * 1000, + secondsToWait(pkg, target, reporter), target.region ), CullInstancesWithTerminationTag(autoScalingGroup, target.region), TerminationGrace( autoScalingGroup, target.region, - terminationGrace(pkg, target, reporter) * 1000 + terminationGrace(pkg, target, reporter) ), WaitForStabilization( autoScalingGroup, - secondsToWait(pkg, target, reporter) * 1000, + secondsToWait(pkg, target, reporter), target.region ), ResumeAlarmNotifications(autoScalingGroup, target.region) diff --git a/magenta-lib/src/main/scala/magenta/deployment_type/CloudFormation.scala b/magenta-lib/src/main/scala/magenta/deployment_type/CloudFormation.scala index 22e91f8be..3ec25caa0 100644 --- a/magenta-lib/src/main/scala/magenta/deployment_type/CloudFormation.scala +++ b/magenta-lib/src/main/scala/magenta/deployment_type/CloudFormation.scala @@ -8,6 +8,9 @@ import magenta.tasks.UpdateCloudFormationTask.LookupByTags import magenta.tasks._ import org.joda.time.DateTime +import java.time.Duration +import java.time.Duration.ofMinutes + trait BuildTags { // Returns tags for a build, which should be added to the Cloudformation // stack. Tags are named with a `gu:` prefix. @@ -91,10 +94,12 @@ class CloudFormation(tagger: BuildTags) "If set to true then the cloudformation stack will be created if it doesn't already exist" ).default(true) - val secondsToWaitForChangeSetCreation = Param( - "secondsToWaitForChangeSetCreation", - "Number of seconds to wait for the change set to be created" - ).default(15 * 60) + val secondsToWaitForChangeSetCreation: Param[Duration] = Param + .waitingSecondsFor( + "secondsToWaitForChangeSetCreation", + "the change set to be created" + ) + .default(ofMinutes(15)) val manageStackPolicyDefault = true val manageStackPolicyLookupKey = "cloudformation:manage-stack-policy" @@ -231,7 +236,7 @@ class CloudFormation(tagger: BuildTags) new CheckChangeSetCreatedTask( target.region, stackLookup, - secondsToWaitForChangeSetCreation(pkg, target, reporter) * 1000 + secondsToWaitForChangeSetCreation(pkg, target, reporter) ), new ExecuteChangeSetTask( target.region, diff --git a/magenta-lib/src/main/scala/magenta/deployment_type/GCS.scala b/magenta-lib/src/main/scala/magenta/deployment_type/GCS.scala index 5a7174c2d..f816101fe 100644 --- a/magenta-lib/src/main/scala/magenta/deployment_type/GCS.scala +++ b/magenta-lib/src/main/scala/magenta/deployment_type/GCS.scala @@ -9,13 +9,22 @@ object GCS extends DeploymentType { val documentation = "For uploading files into a GCS bucket." val prefixStage = - Param("prefixStage", "Prefix the GCS bucket key with the target stage") + Param[Boolean]( + "prefixStage", + "Prefix the GCS bucket key with the target stage" + ) .default(true) val prefixPackage = - Param("prefixPackage", "Prefix the GCS bucket key with the package name") + Param[Boolean]( + "prefixPackage", + "Prefix the GCS bucket key with the package name" + ) .default(true) val prefixStack = - Param("prefixStack", "Prefix the GCS bucket key with the target stack") + Param[Boolean]( + "prefixStack", + "Prefix the GCS bucket key with the target stack" + ) .default(true) val pathPrefixResource = Param[String]( "pathPrefixResource", diff --git a/magenta-lib/src/main/scala/magenta/deployment_type/GcpDeploymentManager.scala b/magenta-lib/src/main/scala/magenta/deployment_type/GcpDeploymentManager.scala index 5f0d9b4b4..a505f68ea 100644 --- a/magenta-lib/src/main/scala/magenta/deployment_type/GcpDeploymentManager.scala +++ b/magenta-lib/src/main/scala/magenta/deployment_type/GcpDeploymentManager.scala @@ -11,7 +11,8 @@ import magenta.tasks.gcp.GCP.DeploymentManagerApi.DeploymentBundle import magenta.{DeployReporter, KeyRing} import software.amazon.awssdk.services.s3.S3Client -import scala.concurrent.duration._ +import java.time.Duration +import java.time.Duration.ofMinutes object GcpDeploymentManager extends DeploymentType { val GCP_PROJECT_NAME_PRISM_KEY: String = "gcp:project-name" @@ -23,12 +24,10 @@ object GcpDeploymentManager extends DeploymentType { | |""".stripMargin - val maxWaitParam: Param[Int] = Param[Int]( - name = "maxWait", - documentation = """ - |Number of seconds to wait for the deployment operations to complete - |""".stripMargin - ).default(1800) // half an hour + val maxWaitParam: Param[Duration] = + Param + .waitingSecondsFor("maxWait", "the deployment operations to complete") + .default(ofMinutes(30)) val deploymentNameParam: Param[String] = Param( name = "deploymentName", @@ -93,7 +92,7 @@ object GcpDeploymentManager extends DeploymentType { } implicit val artifactClient: S3Client = resources.artifactClient - val maxWaitDuration = maxWaitParam(pkg, target, reporter).seconds + val maxWaitDuration = maxWaitParam(pkg, target, reporter) val deploymentName = deploymentNameParam(pkg, target, reporter) val upsert = upsertParam(pkg, target, reporter) val preview = previewParam(pkg, target, reporter) diff --git a/magenta-lib/src/main/scala/magenta/deployment_type/Param.scala b/magenta-lib/src/main/scala/magenta/deployment_type/Param.scala index eba89de66..fde027a3d 100644 --- a/magenta-lib/src/main/scala/magenta/deployment_type/Param.scala +++ b/magenta-lib/src/main/scala/magenta/deployment_type/Param.scala @@ -1,7 +1,9 @@ package magenta.deployment_type import magenta.{DeployReporter, DeployTarget, DeploymentPackage} -import play.api.libs.json.{Json, Reads} +import play.api.libs.json.{JsValue, Json, Reads} + +import java.time.Duration trait ParamRegister { def add(param: Param[_]): Unit @@ -38,21 +40,24 @@ case class Param[T]( (DeploymentPackage, DeployTarget) => Either[String, T] ] = None, deprecatedDefault: Boolean = false -)(implicit register: ParamRegister) { +)(implicit register: ParamRegister, reads: Reads[T], manifest: Manifest[T]) { register.add(this) val required = !optional && defaultValue.isEmpty && defaultValueFromContext.isEmpty - def get(pkg: DeploymentPackage)(implicit reads: Reads[T]): Option[T] = + def get(pkg: DeploymentPackage): Option[T] = pkg.pkgSpecificData .get(name) - .flatMap(jsValue => Json.fromJson[T](jsValue).asOpt) + .flatMap(jsValue => parse(jsValue)) + + def parse(jsValue: JsValue): Option[T] = Json.fromJson[T](jsValue).asOpt + def apply( pkg: DeploymentPackage, target: DeployTarget, reporter: DeployReporter - )(implicit reads: Reads[T], manifest: Manifest[T]): T = { + ): T = { val maybeValue = get(pkg) val defaultFromContext = defaultValueFromContext.map(_(pkg, target)) @@ -96,3 +101,23 @@ case class Param[T]( this.copy(defaultValueFromContext = Some(defaultFromContext)) } } + +object Param { + private val ReadIntAsSeconds: Reads[Duration] = + Reads.of[Int].map(Duration.ofSeconds(_)) + + /** Create a parameter that represents a number of seconds to wait for + * something. Riff Raff has many existing parameters for setting durations + * where the values are *expected* to be in seconds, so this helper method + * makes that clear and ensures that the value is explicitly parsed as a + * duration in *seconds*. + */ + def waitingSecondsFor(name: String, waitingOn: String)(implicit + register: ParamRegister + ): Param[Duration] = + Param[Duration](name, s"Number of seconds to wait for $waitingOn")( + register, + ReadIntAsSeconds, + implicitly[Manifest[Duration]] + ) +} diff --git a/magenta-lib/src/main/scala/magenta/deployment_type/S3ObjectPrefixParameters.scala b/magenta-lib/src/main/scala/magenta/deployment_type/S3ObjectPrefixParameters.scala index ca68694ac..a6c61c350 100644 --- a/magenta-lib/src/main/scala/magenta/deployment_type/S3ObjectPrefixParameters.scala +++ b/magenta-lib/src/main/scala/magenta/deployment_type/S3ObjectPrefixParameters.scala @@ -7,15 +7,24 @@ trait S3ObjectPrefixParameters { this: DeploymentType => val prefixStage: Param[Boolean] = - Param("prefixStage", "Prefix the S3 bucket key with the target stage") + Param[Boolean]( + "prefixStage", + "Prefix the S3 bucket key with the target stage" + ) .default(true) val prefixPackage: Param[Boolean] = - Param("prefixPackage", "Prefix the S3 bucket key with the package name") + Param[Boolean]( + "prefixPackage", + "Prefix the S3 bucket key with the package name" + ) .default(true) val prefixStack: Param[Boolean] = - Param("prefixStack", "Prefix the S3 bucket key with the target stack") + Param[Boolean]( + "prefixStack", + "Prefix the S3 bucket key with the target stack" + ) .default(true) val prefixApp: Param[Boolean] = Param[Boolean]( diff --git a/magenta-lib/src/main/scala/magenta/tasks/ASGTasks.scala b/magenta-lib/src/main/scala/magenta/tasks/ASGTasks.scala index 6c7e97984..f998d6d71 100644 --- a/magenta-lib/src/main/scala/magenta/tasks/ASGTasks.scala +++ b/magenta-lib/src/main/scala/magenta/tasks/ASGTasks.scala @@ -9,8 +9,8 @@ import software.amazon.awssdk.services.autoscaling.model.{ LifecycleState, SetInstanceProtectionRequest } -import software.amazon.awssdk.services.ec2.Ec2Client +import java.time.Duration import scala.jdk.CollectionConverters._ case class CheckGroupSize(info: AutoScalingGroupInfo, region: Region)(implicit @@ -118,8 +118,12 @@ case class DoubleSize(info: AutoScalingGroupInfo, region: Region)(implicit s"Double the size of the auto-scaling group called $asgName" } -sealed abstract class Pause(durationMillis: Long)(implicit val keyRing: KeyRing) +sealed abstract class Pause(duration: Duration)(implicit val keyRing: KeyRing) extends ASGTask { + val purpose: String + + def description: String = s"Wait extra ${duration.toMillis}ms to $purpose" + def execute( asg: AutoScalingGroup, resources: DeploymentResources, @@ -131,43 +135,40 @@ sealed abstract class Pause(durationMillis: Long)(implicit val keyRing: KeyRing) "Skipping pause as there are no instances and desired capacity is zero" ) else - Thread.sleep(durationMillis) + Thread.sleep(duration.toMillis) } } case class HealthcheckGrace( info: AutoScalingGroupInfo, region: Region, - durationMillis: Long + duration: Duration )(implicit keyRing: KeyRing) - extends Pause(durationMillis) { - def description: String = - s"Wait extra ${durationMillis}ms to let Load Balancer report correctly" + extends Pause(duration) { + val purpose: String = "let Load Balancer report correctly" } case class WarmupGrace( info: AutoScalingGroupInfo, region: Region, - durationMillis: Long + duration: Duration )(implicit keyRing: KeyRing) - extends Pause(durationMillis) { - def description: String = - s"Wait extra ${durationMillis}ms to let instances in Load Balancer warm up" + extends Pause(duration) { + val purpose: String = "let instances in Load Balancer warm up" } case class TerminationGrace( info: AutoScalingGroupInfo, region: Region, - durationMillis: Long + duration: Duration )(implicit keyRing: KeyRing) - extends Pause(durationMillis) { - def description: String = - s"Wait extra ${durationMillis}ms to let Load Balancer report correctly" + extends Pause(duration) { + val purpose: String = "let Load Balancer report correctly" } case class WaitForStabilization( info: AutoScalingGroupInfo, - duration: Long, + duration: Duration, region: Region )(implicit val keyRing: KeyRing) extends ASGTask diff --git a/magenta-lib/src/main/scala/magenta/tasks/ChangeSetTasks.scala b/magenta-lib/src/main/scala/magenta/tasks/ChangeSetTasks.scala index 68c881939..9193f9abc 100644 --- a/magenta-lib/src/main/scala/magenta/tasks/ChangeSetTasks.scala +++ b/magenta-lib/src/main/scala/magenta/tasks/ChangeSetTasks.scala @@ -13,6 +13,7 @@ import software.amazon.awssdk.services.cloudformation.model.ChangeSetStatus._ import software.amazon.awssdk.services.cloudformation.model._ import software.amazon.awssdk.services.s3.S3Client +import java.time.Duration import scala.jdk.CollectionConverters._ import scala.util.{Success, Try} @@ -150,7 +151,7 @@ class CreateChangeSetTask( class CheckChangeSetCreatedTask( region: Region, stackLookup: CloudFormationStackMetadata, - override val duration: Long + override val duration: Duration )(implicit val keyRing: KeyRing, artifactClient: S3Client) extends Task with RepeatedPollingCheck { diff --git a/magenta-lib/src/main/scala/magenta/tasks/UpdateCloudFormationTask.scala b/magenta-lib/src/main/scala/magenta/tasks/UpdateCloudFormationTask.scala index bc4b5ff58..136b6b6ed 100644 --- a/magenta-lib/src/main/scala/magenta/tasks/UpdateCloudFormationTask.scala +++ b/magenta-lib/src/main/scala/magenta/tasks/UpdateCloudFormationTask.scala @@ -18,7 +18,6 @@ import magenta.{ Stack, Stage } -import org.joda.time.{DateTime, Duration} import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.services.cloudformation.CloudFormationClient import software.amazon.awssdk.services.cloudformation.model.{ @@ -31,21 +30,23 @@ import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model.PutObjectRequest import software.amazon.awssdk.services.sts.StsClient +import java.time.{Duration, Instant} +import java.time.Duration.{between, ofMinutes, ofSeconds} import scala.annotation.tailrec import scala.jdk.CollectionConverters._ +import scala.math.Ordering.Implicits._ /** A simple trait to aid with attempting an update multiple times in the case * that an update is already running. */ trait RetryCloudFormationUpdate { - def duration: Long = 15 * 60 * 1000 // wait fifteen minutes - def calculateSleepTime(currentAttempt: Int): Long = - 30 * 1000 // sleep 30 seconds + def duration: Duration = ofMinutes(15) + def calculateSleepTime(currentAttempt: Int): Duration = ofSeconds(30) def updateWithRetry[T](reporter: DeployReporter, stopFlag: => Boolean)( theUpdate: => T ): Option[T] = { - val expiry = System.currentTimeMillis() + duration + val expiry = Instant.now().plus(duration) def updateAttempt(currentAttempt: Int): Option[T] = { try { @@ -62,18 +63,18 @@ trait RetryCloudFormationUpdate { ) None } else { - val remainingTime = expiry - System.currentTimeMillis() - if (remainingTime > 0) { + val remainingTime = between(Instant.now(), expiry) + if (remainingTime.isNegative) + reporter.fail( + s"Update is still running after $duration milliseconds (tried $currentAttempt times) - aborting" + ) + else { val sleepyTime = calculateSleepTime(currentAttempt) reporter.verbose( - f"Another update is running against this cloudformation stack, waiting for it to finish (tried $currentAttempt%s, will try again in ${sleepyTime.toFloat / 1000}%.1f, will give up in ${remainingTime.toFloat / 1000}%.1f)" + f"Another update is running against this cloudformation stack, waiting for it to finish (Will wait for a further ${remainingTime.toSeconds} seconds, retrying again after ${sleepyTime.toSeconds}s)" ) - Thread.sleep(sleepyTime) + Thread.sleep(sleepyTime.toMillis) updateAttempt(currentAttempt + 1) - } else { - reporter.fail( - s"Update is still running after $duration milliseconds (tried $currentAttempt times) - aborting" - ) } } case e: CloudFormationException => @@ -383,7 +384,7 @@ object UpdateCloudFormationTask extends Loggable { reporter, Some(1) ) - val keyName = s"$stackName-${new DateTime().getMillis}" + val keyName = s"$stackName-${System.currentTimeMillis()}" reporter.verbose( s"Uploading template as $keyName to S3 bucket $bucketName" ) @@ -609,11 +610,8 @@ class CloudFormationStackEventPoller( s"No events found at all for stack $stackName" ) case Some(e) => - val age = new Duration( - new DateTime(e.timestamp().toEpochMilli), - new DateTime() - ).getStandardSeconds - if (age > 30) { + val age = Duration.between(e.timestamp(), Instant.now()) + if (age > ofSeconds(30)) { resources.reporter.verbose( "No recent IN_PROGRESS events found (nothing within last 30 seconds)" ) diff --git a/magenta-lib/src/main/scala/magenta/tasks/gcp/DeploymentManagerTasks.scala b/magenta-lib/src/main/scala/magenta/tasks/gcp/DeploymentManagerTasks.scala index 3a1fcc0bb..868dadd32 100644 --- a/magenta-lib/src/main/scala/magenta/tasks/gcp/DeploymentManagerTasks.scala +++ b/magenta-lib/src/main/scala/magenta/tasks/gcp/DeploymentManagerTasks.scala @@ -7,14 +7,15 @@ import magenta.tasks.{PollingCheck, Task} import magenta.tasks.gcp.GCP.DeploymentManagerApi._ import magenta.tasks.gcp.GCPRetryHelper.Result -import scala.concurrent.duration.FiniteDuration +import java.time.Duration +import java.time.Duration.ofSeconds object DeploymentManagerTasks { def updateTask( project: String, deploymentName: String, bundle: DeploymentBundle, - maxWait: FiniteDuration, + maxWait: Duration, upsert: Boolean, preview: Boolean )(implicit kr: KeyRing): Task = new Task with PollingCheck { @@ -127,8 +128,9 @@ object DeploymentManagerTasks { } } - override def duration: Long = maxWait.toMillis + override def duration: Duration = maxWait - override def calculateSleepTime(currentAttempt: Int): Long = 5000 + override def calculateSleepTime(currentAttempt: Int): Duration = + ofSeconds(5) } } diff --git a/magenta-lib/src/main/scala/magenta/tasks/tasks.scala b/magenta-lib/src/main/scala/magenta/tasks/tasks.scala index ab0275a18..f8b7dccf6 100644 --- a/magenta-lib/src/main/scala/magenta/tasks/tasks.scala +++ b/magenta-lib/src/main/scala/magenta/tasks/tasks.scala @@ -1,16 +1,14 @@ package magenta package tasks -import java.io.{File, InputStream, PipedInputStream, PipedOutputStream} import magenta.artifact._ +import magenta.deployment_type.param_reads.PatternValue import magenta.deployment_type.{ LambdaFunction, LambdaFunctionName, LambdaFunctionTags, - LambdaLayer, LambdaLayerName } -import magenta.deployment_type.param_reads.PatternValue import okhttp3.{FormBody, HttpUrl, OkHttpClient, Request} import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration import software.amazon.awssdk.core.internal.util.Mimetype @@ -18,21 +16,20 @@ import software.amazon.awssdk.core.sync.{ ResponseTransformer, RequestBody => AWSRequestBody } -import software.amazon.awssdk.http.ContentStreamProvider import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model.{ GetObjectRequest, GetObjectResponse, - HeadObjectRequest, ObjectCannedACL, PutObjectRequest } +import java.io.{File, PipedInputStream, PipedOutputStream} +import java.time.Duration.{between, ofMillis, ofSeconds} +import java.time.{Duration, Instant} +import scala.collection.parallel.CollectionConverters._ import scala.concurrent.{Await, Future} -import scala.concurrent.duration._ -import scala.concurrent.ExecutionContext.Implicits.global import scala.util.control.NonFatal -import scala.collection.parallel.CollectionConverters._ case class S3Upload( region: Region, @@ -163,6 +160,7 @@ case class S3Upload( is.close() } + import scala.concurrent.duration._ Await.result(response, 5 minutes) logger.debug( @@ -269,51 +267,51 @@ case class PutReq( } trait PollingCheck { - def duration: Long + def duration: Duration def check(reporter: DeployReporter, stopFlag: => Boolean)( theCheck: => Boolean ): Unit = { - val expiry = System.currentTimeMillis() + duration - - def checkAttempt(currentAttempt: Int): Unit = { - if (!theCheck) { - if (stopFlag) { - reporter.info("Abandoning remaining checks as stop flag has been set") - } else { - val remainingTime = expiry - System.currentTimeMillis() - if (remainingTime > 0) { - val sleepyTime = calculateSleepTime(currentAttempt) - reporter.verbose( - f"Check failed on attempt #$currentAttempt (Will wait for a further ${remainingTime.toFloat / 1000} seconds, retrying again after ${sleepyTime.toFloat / 1000}s)" - ) - Thread.sleep(sleepyTime) - checkAttempt(currentAttempt + 1) - } else { - reporter.fail( - s"Check failed to pass within $duration milliseconds (tried $currentAttempt times) - aborting" - ) - } + val expiry = Instant.now().plus(duration) + + def checkAttempt(currentAttempt: Int): Unit = if (!theCheck) { + if (stopFlag) + reporter.info("Abandoning remaining checks as stop flag has been set") + else { + val remainingTime = between(Instant.now(), expiry) + if (remainingTime.isNegative) + reporter.fail( + s"Check failed to pass within $duration milliseconds (tried $currentAttempt times) - aborting" + ) + else { + val sleepyTime = calculateSleepTime(currentAttempt) + reporter.verbose( + f"Check failed on attempt #$currentAttempt (Will wait for a further ${remainingTime.toSeconds} seconds, retrying again after ${sleepyTime.toSeconds}s)" + ) + Thread.sleep(sleepyTime.toMillis) + checkAttempt(currentAttempt + 1) } } } + checkAttempt(1) } - def calculateSleepTime(currentAttempt: Int): Long + def calculateSleepTime(currentAttempt: Int): Duration } trait RepeatedPollingCheck extends PollingCheck { - def calculateSleepTime(currentAttempt: Int): Long = { + def calculateSleepTime(currentAttempt: Int): Duration = { val exponent = math.min(currentAttempt, 8) - math.min(math.pow(2, exponent).toLong * 100, 25000) + ofMillis(math.min(math.pow(2, exponent).toLong * 100, 25000)) } } trait SlowRepeatedPollingCheck extends PollingCheck { - def calculateSleepTime(currentAttempt: Int): Long = 30000 + def calculateSleepTime(currentAttempt: Int): Duration = + ofSeconds(30) } case class SayHello(host: Host)(implicit val keyRing: KeyRing) extends Task { diff --git a/magenta-lib/src/test/scala/magenta/deployment_type/AutoScalingTest.scala b/magenta-lib/src/test/scala/magenta/deployment_type/AutoScalingTest.scala index e6a304ec9..2bfeb4e69 100644 --- a/magenta-lib/src/test/scala/magenta/deployment_type/AutoScalingTest.scala +++ b/magenta-lib/src/test/scala/magenta/deployment_type/AutoScalingTest.scala @@ -20,6 +20,7 @@ import magenta.tasks.ASG.{TagAbsent, TagExists, TagMatch} import magenta.tasks._ import org.mockito.ArgumentMatchersSugar import org.mockito.MockitoSugar +import org.scalatest.{Inspectors, OptionValues} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import play.api.libs.json.{JsBoolean, JsNumber, JsString, JsValue} @@ -27,11 +28,14 @@ import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.sts.StsClient +import java.time.Duration.{ofMinutes, ofSeconds} import scala.concurrent.ExecutionContext.global class AutoScalingTest extends AnyFlatSpec with Matchers + with Inspectors + with OptionValues with MockitoSugar with ArgumentMatchersSugar { implicit val fakeKeyRing: KeyRing = KeyRing() @@ -51,6 +55,15 @@ class AutoScalingTest AutoScalingGroupInfo(asg, List(TagMatch("App", name))) } + "AutoScaling parameters for durations" should "behave as documented and assume integers are in seconds" in { + import AutoScaling._ + forAll( + Seq(secondsToWait, healthcheckGrace, warmupGrace, terminationGrace) + ) { param => + param.parse(JsNumber(30)).value shouldEqual ofSeconds(30) + } + } + "AutoScalingGroupLookup.getTagRequirements" should "return the right tags for a basic app" in { val tagReqs = AutoScalingGroupLookup.getTagRequirements( Stage("testStage"), @@ -130,19 +143,19 @@ class AutoScalingTest DeployTarget(parameters(), stack, region) ) val expected = List( - WaitForStabilization(testInfo, 5 * 60 * 1000, Region("eu-west-1")), + WaitForStabilization(testInfo, ofMinutes(5), Region("eu-west-1")), CheckGroupSize(testInfo, Region("eu-west-1")), SuspendAlarmNotifications(testInfo, Region("eu-west-1")), TagCurrentInstancesWithTerminationTag(testInfo, Region("eu-west-1")), ProtectCurrentInstances(testInfo, Region("eu-west-1")), DoubleSize(testInfo, Region("eu-west-1")), - HealthcheckGrace(testInfo, Region("eu-west-1"), 20000), - WaitForStabilization(testInfo, 15 * 60 * 1000, Region("eu-west-1")), - WarmupGrace(testInfo, Region("eu-west-1"), 1000), - WaitForStabilization(testInfo, 15 * 60 * 1000, Region("eu-west-1")), + HealthcheckGrace(testInfo, Region("eu-west-1"), ofSeconds(20)), + WaitForStabilization(testInfo, ofMinutes(15), Region("eu-west-1")), + WarmupGrace(testInfo, Region("eu-west-1"), ofSeconds(1)), + WaitForStabilization(testInfo, ofMinutes(15), Region("eu-west-1")), CullInstancesWithTerminationTag(testInfo, Region("eu-west-1")), - TerminationGrace(testInfo, Region("eu-west-1"), 10000), - WaitForStabilization(testInfo, 15 * 60 * 1000, Region("eu-west-1")), + TerminationGrace(testInfo, Region("eu-west-1"), ofSeconds(10)), + WaitForStabilization(testInfo, ofMinutes(15), Region("eu-west-1")), ResumeAlarmNotifications(testInfo, Region("eu-west-1")) ) actual shouldBe expected @@ -186,7 +199,7 @@ class AutoScalingTest ) val expected = List( // All tasks for testOldCfnAsg - WaitForStabilization(testOldCfnAsg, 5 * 60 * 1000, Region("eu-west-1")), + WaitForStabilization(testOldCfnAsg, ofMinutes(5), Region("eu-west-1")), CheckGroupSize(testOldCfnAsg, Region("eu-west-1")), SuspendAlarmNotifications(testOldCfnAsg, Region("eu-west-1")), TagCurrentInstancesWithTerminationTag( @@ -195,28 +208,28 @@ class AutoScalingTest ), ProtectCurrentInstances(testOldCfnAsg, Region("eu-west-1")), DoubleSize(testOldCfnAsg, Region("eu-west-1")), - HealthcheckGrace(testOldCfnAsg, Region("eu-west-1"), 20000), + HealthcheckGrace(testOldCfnAsg, Region("eu-west-1"), ofSeconds(20)), WaitForStabilization( testOldCfnAsg, - 15 * 60 * 1000, + ofMinutes(15), Region("eu-west-1") ), - WarmupGrace(testOldCfnAsg, Region("eu-west-1"), 1000), + WarmupGrace(testOldCfnAsg, Region("eu-west-1"), ofSeconds(1)), WaitForStabilization( testOldCfnAsg, - 15 * 60 * 1000, + ofMinutes(15), Region("eu-west-1") ), CullInstancesWithTerminationTag(testOldCfnAsg, Region("eu-west-1")), - TerminationGrace(testOldCfnAsg, Region("eu-west-1"), 10000), + TerminationGrace(testOldCfnAsg, Region("eu-west-1"), ofSeconds(10)), WaitForStabilization( testOldCfnAsg, - 15 * 60 * 1000, + ofMinutes(15), Region("eu-west-1") ), ResumeAlarmNotifications(testOldCfnAsg, Region("eu-west-1")), // All tasks for testNewCdkAsg - WaitForStabilization(testNewCdkAsg, 5 * 60 * 1000, Region("eu-west-1")), + WaitForStabilization(testNewCdkAsg, ofMinutes(5), Region("eu-west-1")), CheckGroupSize(testNewCdkAsg, Region("eu-west-1")), SuspendAlarmNotifications(testNewCdkAsg, Region("eu-west-1")), TagCurrentInstancesWithTerminationTag( @@ -225,23 +238,23 @@ class AutoScalingTest ), ProtectCurrentInstances(testNewCdkAsg, Region("eu-west-1")), DoubleSize(testNewCdkAsg, Region("eu-west-1")), - HealthcheckGrace(testNewCdkAsg, Region("eu-west-1"), 20000), + HealthcheckGrace(testNewCdkAsg, Region("eu-west-1"), ofSeconds(20)), WaitForStabilization( testNewCdkAsg, - 15 * 60 * 1000, + ofMinutes(15), Region("eu-west-1") ), - WarmupGrace(testNewCdkAsg, Region("eu-west-1"), 1000), + WarmupGrace(testNewCdkAsg, Region("eu-west-1"), ofSeconds(1)), WaitForStabilization( testNewCdkAsg, - 15 * 60 * 1000, + ofMinutes(15), Region("eu-west-1") ), CullInstancesWithTerminationTag(testNewCdkAsg, Region("eu-west-1")), - TerminationGrace(testNewCdkAsg, Region("eu-west-1"), 10000), + TerminationGrace(testNewCdkAsg, Region("eu-west-1"), ofSeconds(10)), WaitForStabilization( testNewCdkAsg, - 15 * 60 * 1000, + ofMinutes(15), Region("eu-west-1") ), ResumeAlarmNotifications(testNewCdkAsg, Region("eu-west-1")) @@ -322,19 +335,19 @@ class AutoScalingTest DeployTarget(parameters(), stack, region) ) val expected = List( - WaitForStabilization(testInfo, 5 * 60 * 1000, Region("eu-west-1")), + WaitForStabilization(testInfo, ofMinutes(5), Region("eu-west-1")), CheckGroupSize(testInfo, Region("eu-west-1")), SuspendAlarmNotifications(testInfo, Region("eu-west-1")), TagCurrentInstancesWithTerminationTag(testInfo, Region("eu-west-1")), ProtectCurrentInstances(testInfo, Region("eu-west-1")), DoubleSize(testInfo, Region("eu-west-1")), - HealthcheckGrace(testInfo, Region("eu-west-1"), 30000), - WaitForStabilization(testInfo, 3 * 60 * 1000, Region("eu-west-1")), - WarmupGrace(testInfo, Region("eu-west-1"), 20000), - WaitForStabilization(testInfo, 3 * 60 * 1000, Region("eu-west-1")), + HealthcheckGrace(testInfo, Region("eu-west-1"), ofSeconds(30)), + WaitForStabilization(testInfo, ofMinutes(3), Region("eu-west-1")), + WarmupGrace(testInfo, Region("eu-west-1"), ofSeconds(20)), + WaitForStabilization(testInfo, ofMinutes(3), Region("eu-west-1")), CullInstancesWithTerminationTag(testInfo, Region("eu-west-1")), - TerminationGrace(testInfo, Region("eu-west-1"), 11000), - WaitForStabilization(testInfo, 3 * 60 * 1000, Region("eu-west-1")), + TerminationGrace(testInfo, Region("eu-west-1"), ofSeconds(11)), + WaitForStabilization(testInfo, ofMinutes(3), Region("eu-west-1")), ResumeAlarmNotifications(testInfo, Region("eu-west-1")) ) actual shouldBe expected diff --git a/magenta-lib/src/test/scala/magenta/deployment_type/CloudFormationTest.scala b/magenta-lib/src/test/scala/magenta/deployment_type/CloudFormationTest.scala index 2fdc7bbd0..79f264404 100644 --- a/magenta-lib/src/test/scala/magenta/deployment_type/CloudFormationTest.scala +++ b/magenta-lib/src/test/scala/magenta/deployment_type/CloudFormationTest.scala @@ -16,8 +16,8 @@ import magenta.tasks.UpdateCloudFormationTask._ import magenta.tasks._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import org.scalatest.{EitherValues, Inside} -import play.api.libs.json.{JsBoolean, JsString, JsValue, Json} +import org.scalatest.{EitherValues, Inside, OptionValues} +import play.api.libs.json.{JsBoolean, JsNumber, JsString, JsValue, Json} import software.amazon.awssdk.services.cloudformation.model.{ Change, ChangeSetType @@ -25,6 +25,7 @@ import software.amazon.awssdk.services.cloudformation.model.{ import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.sts.StsClient +import java.time.Duration.ofSeconds import java.util.UUID import scala.concurrent.ExecutionContext.global @@ -35,6 +36,7 @@ class CloudFormationTest extends AnyFlatSpec with Matchers with Inside + with OptionValues with EitherValues { implicit val fakeKeyRing: KeyRing = KeyRing() implicit val reporter: DeployReporter = @@ -87,6 +89,14 @@ class CloudFormationTest ) } + "parameters for durations" should "behave as documented and assume integers are in seconds" in { + cloudformationDeploymentType.secondsToWaitForChangeSetCreation + .parse( + JsNumber(30) + ) + .value shouldEqual ofSeconds(30) + } + it should "generate the tasks in the correct order when manageStackPolicy is false" in { val data: Map[String, JsValue] = Map( "manageStackPolicy" -> JsBoolean(false) diff --git a/magenta-lib/src/test/scala/magenta/deployment_type/ParamTest.scala b/magenta-lib/src/test/scala/magenta/deployment_type/ParamTest.scala index bd0501aae..170b7512a 100644 --- a/magenta-lib/src/test/scala/magenta/deployment_type/ParamTest.scala +++ b/magenta-lib/src/test/scala/magenta/deployment_type/ParamTest.scala @@ -1,20 +1,31 @@ package magenta.deployment_type import java.util.UUID - import magenta.artifact.S3Path import magenta._ import magenta.fixtures._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import org.mockito.MockitoSugar -import play.api.libs.json.JsString +import play.api.libs.json.{JsNumber, JsString, JsValue} +import java.time.Duration.ofSeconds import scala.collection.mutable -class TestRegister extends ParamRegister { - val paramsList = mutable.Map.empty[String, Param[_]] - def add(param: Param[_]) = paramsList += param.name -> param +class ParamRegistrationTest extends AnyFlatSpec with Matchers { + class TestRegister extends ParamRegister { + val paramsList = mutable.Map.empty[String, Param[_]] + def add(param: Param[_]) = paramsList += param.name -> param + } + + "Param registration" should "occur when a param is created" in { + implicit val register: TestRegister = new TestRegister + + val param = Param[String]("test") + + register.paramsList.size shouldBe 1 + register.paramsList shouldBe Map("test" -> param) + } } class ParamTest extends AnyFlatSpec with Matchers with MockitoSugar { @@ -31,165 +42,100 @@ class ParamTest extends AnyFlatSpec with Matchers with MockitoSugar { actionNames = Seq("testAction") ) ) - - "Param" should "register itself with a register" in { - implicit val register = new TestRegister - - val param = Param[String]("test") - - register.paramsList.size shouldBe 1 - register.paramsList shouldBe Map("test" -> param) - } - - it should "extract a value from a package using get" in { - implicit val register = new TestRegister - val pkg = DeploymentPackage( + def stubDeploymentPackage(pkgSpecificData: Map[String, JsValue]) = + DeploymentPackage( "testPackage", app1, - Map("key" -> JsString("myValue")), + pkgSpecificData, "testDeploymentType", S3Path("test", "test"), deploymentTypes ) - val key = Param[String]("key").default("valueDefault") - val paramValue = key.get(pkg) + val Key = "demoParamName" + private val pkgWithEmptyConfig: DeploymentPackage = + stubDeploymentPackage(Map.empty) + + implicit val stubRegister: ParamRegister = + (_: Param[_]) => + () // We don't care about param registration for these tests + + "Param" should "treat values as seconds when we've been explicit about it" in { + val key = Param + .waitingSecondsFor(Key, "something important to happen") + .default(ofSeconds(7)) + key.documentation shouldBe "Number of seconds to wait for something important to happen" + key.get(stubDeploymentPackage(Map(Key -> JsNumber(15)))) shouldBe Some( + ofSeconds(15) + ) + } + + it should "extract a value from a package using get" in { + val param = Param[String](Key).default("valueDefault") + val paramValue = + param.get(stubDeploymentPackage(Map(Key -> JsString("myValue")))) paramValue shouldBe Some("myValue") } it should "extract None using get when the value isn't in a package" in { - implicit val register = new TestRegister - val pkg = DeploymentPackage( - "testPackage", - app1, - Map.empty, - "testDeploymentType", - S3Path("test", "test"), - deploymentTypes - ) - val key = Param[String]("key").default("valueDefault") - val paramValue = key.get(pkg) - paramValue shouldBe None + Param[String](Key).default("d").get(pkgWithEmptyConfig) shouldBe None } it should "extract a value using apply" in { - implicit val register = new TestRegister - val pkg = DeploymentPackage( - "testPackage", - app1, - Map("key" -> JsString("myValue")), - "testDeploymentType", - S3Path("test", "test"), - deploymentTypes - ) - val key = Param[String]("key").default("valueDefault") - val paramValue = key.apply(pkg, target, reporter) - paramValue shouldBe "myValue" + val pkg = stubDeploymentPackage(Map(Key -> JsString("myValue"))) + val param = Param[String](Key).default("valueDefault") + param.apply(pkg, target, reporter) shouldBe "myValue" } it should "throw an exception if a value is not specified and has no default" in { - implicit val register = new TestRegister - val pkg = DeploymentPackage( - "testPackage", - app1, - Map.empty, - "testDeploymentType", - S3Path("test", "test"), - deploymentTypes - ) - val key = Param[String]("key") + val param = Param[String](Key) val thrown = the[NoSuchElementException] thrownBy { - key.apply(pkg, target, reporter) + param.apply(pkgWithEmptyConfig, target, reporter) } - thrown.getMessage shouldBe "Package testPackage [testDeploymentType] requires parameter key of type String" + thrown.getMessage shouldBe s"Package testPackage [testDeploymentType] requires parameter $Key of type String" } it should "return the param default from apply when no value is specified" in { - implicit val register = new TestRegister - val pkg = DeploymentPackage( - "testPackage", - app1, - Map.empty, - "testDeploymentType", - S3Path("test", "test"), - deploymentTypes - ) - val key = Param[String]("key").default("valueDefault") - val paramValue = key.apply(pkg, target, reporter) - paramValue shouldBe "valueDefault" + val param = Param[String](Key).default("valueDefault") + param.apply(pkgWithEmptyConfig, target, reporter) shouldBe "valueDefault" } it should "return the param context default from apply when no value is specified" in { - implicit val register = new TestRegister - val pkg = DeploymentPackage( - "testPackage", - app1, - Map.empty, - "testDeploymentType", - S3Path("test", "test"), - deploymentTypes - ) - val key = Param[String]("key").defaultFromContext((pkg, target) => + val param = Param[String](Key).defaultFromContext((_, target) => Right(s"${target.region.name}") ) - val paramValue = key.apply(pkg, target, reporter) - paramValue shouldBe "testRegion" + val paramValue = param.apply(pkgWithEmptyConfig, target, reporter) + paramValue shouldBe target.region.name } it should "throw an exception if defaultFromContext returns a Left value" in { - implicit val register = new TestRegister - val pkg = DeploymentPackage( - "testPackage", - app1, - Map.empty, - "testDeploymentType", - S3Path("test", "test"), - deploymentTypes - ) - val key = Param[String]("key").defaultFromContext((pkg, target) => + val key = Param[String](Key).defaultFromContext((_, _) => Left("something was wrong") ) val thrown = the[NoSuchElementException] thrownBy { - key.apply(pkg, target, reporter) + key.apply(pkgWithEmptyConfig, target, reporter) } - thrown.getMessage shouldBe "Error whilst generating default for parameter key in package testPackage [testDeploymentType]: something was wrong" + thrown.getMessage shouldBe s"Error whilst generating default for parameter $Key in package testPackage [testDeploymentType]: something was wrong" } it should "log if the value you've specified is the same as the default" in { val mockReporter = mock[DeployReporter] - implicit val register = new TestRegister - val pkg = DeploymentPackage( - "testPackage", - app1, - Map("key" -> JsString("sameValue")), - "testDeploymentType", - S3Path("test", "test"), - deploymentTypes - ) - val key = Param[String]("key").default("sameValue") - val paramValue = key.apply(pkg, target, mockReporter) + val pkg = stubDeploymentPackage(Map(Key -> JsString("sameValue"))) + val paramValue = + Param[String](Key).default("sameValue").apply(pkg, target, mockReporter) paramValue shouldBe "sameValue" verify(mockReporter).info( - "Parameter key is unnecessarily explicitly set to the default value of sameValue" + s"Parameter $Key is unnecessarily explicitly set to the default value of sameValue" ) } it should "log if the value you've specified is the same as the default from context" in { val mockReporter = mock[DeployReporter] - implicit val register = new TestRegister - val pkg = DeploymentPackage( - "testPackage", - app1, - Map("key" -> JsString("sameValue")), - "testDeploymentType", - S3Path("test", "test"), - deploymentTypes - ) - val key = - Param[String]("key").defaultFromContext((_, _) => Right("sameValue")) - val paramValue = key.apply(pkg, target, mockReporter) - paramValue shouldBe "sameValue" + val pkg = stubDeploymentPackage(Map(Key -> JsString("sameValue"))) + val param = + Param[String](Key).defaultFromContext((_, _) => Right("sameValue")) + param.apply(pkg, target, mockReporter) shouldBe "sameValue" verify(mockReporter).info( - "Parameter key is unnecessarily explicitly set to the default value of sameValue" + s"Parameter $Key is unnecessarily explicitly set to the default value of sameValue" ) } } diff --git a/magenta-lib/src/test/scala/magenta/input/resolver/DeploymentTypeResolverTest.scala b/magenta-lib/src/test/scala/magenta/input/resolver/DeploymentTypeResolverTest.scala index f9d483bc3..e7fb16345 100644 --- a/magenta-lib/src/test/scala/magenta/input/resolver/DeploymentTypeResolverTest.scala +++ b/magenta-lib/src/test/scala/magenta/input/resolver/DeploymentTypeResolverTest.scala @@ -95,7 +95,7 @@ class DeploymentTypeResolverTest val deploymentTypesWithParams = List( stubDeploymentType( Seq("upload", "deploy"), - register => List(Param[String]("param1")(register)) + implicit register => List(Param[String]("param1")) ) ) val configErrors = DeploymentTypeResolver @@ -111,7 +111,7 @@ class DeploymentTypeResolverTest val deploymentTypesWithParams = List( stubDeploymentType( Seq("upload", "deploy"), - register => List(Param[String]("param1", optional = true)(register)) + implicit register => List(Param[String]("param1", optional = true)) ) ) DeploymentTypeResolver @@ -123,11 +123,9 @@ class DeploymentTypeResolverTest val deploymentTypesWithParams = List( stubDeploymentType( Seq("upload", "deploy"), - register => + implicit register => List( - Param[String]("param1", defaultValue = Some("defaultValue"))( - register - ) + Param[String]("param1", defaultValue = Some("defaultValue")) ) ) ) @@ -140,12 +138,12 @@ class DeploymentTypeResolverTest val deploymentTypesWithParams = List( stubDeploymentType( Seq("upload", "deploy"), - register => + implicit register => List( Param[String]( "param1", defaultValueFromContext = Some((pkg, _) => Right(pkg.name)) - )(register) + ) ) ) )