From dc2cfb4d0f2b170a16f2ef44b047aea06e00e0a0 Mon Sep 17 00:00:00 2001 From: Robert Kenny Date: Fri, 26 Jan 2024 16:58:58 +0000 Subject: [PATCH 1/4] Bump the SBT version from 1.4.1 to 1.9.8 --- project/build.properties | 2 +- project/build.sbt | 2 +- project/plugins.sbt | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/project/build.properties b/project/build.properties index 08e4d79332..abbbce5da4 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.4.1 +sbt.version=1.9.8 diff --git a/project/build.sbt b/project/build.sbt index 1c3d07424c..47538726bf 100644 --- a/project/build.sbt +++ b/project/build.sbt @@ -1 +1 @@ -scalaVersion := "2.12.7" +scalaVersion := "2.12.17" diff --git a/project/plugins.sbt b/project/plugins.sbt index fdea826eec..75b3aa86d3 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,6 @@ + addSbtPlugin("com.tapad" % "sbt-docker-compose" % "1.0.34") addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.5") addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "1.15") addSbtPlugin("com.frugalmechanic" % "fm-sbt-s3-resolver" % "0.19.0") +addDependencyTreePlugin \ No newline at end of file From 8f2a3092476a007a76bf8d9eb4debe8e90af20b7 Mon Sep 17 00:00:00 2001 From: Robert Kenny Date: Fri, 26 Jan 2024 17:07:03 +0000 Subject: [PATCH 2/4] tedious transitive dependency fiddling --- project/build.sbt | 2 +- project/plugins.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project/build.sbt b/project/build.sbt index 47538726bf..089c89e3d7 100644 --- a/project/build.sbt +++ b/project/build.sbt @@ -1 +1 @@ -scalaVersion := "2.12.17" +scalaVersion := "2.12.15" diff --git a/project/plugins.sbt b/project/plugins.sbt index 75b3aa86d3..87771a49bf 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,6 +1,6 @@ addSbtPlugin("com.tapad" % "sbt-docker-compose" % "1.0.34") -addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.5") +addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.7.5") addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "1.15") addSbtPlugin("com.frugalmechanic" % "fm-sbt-s3-resolver" % "0.19.0") addDependencyTreePlugin \ No newline at end of file From e33c1c78dcb91b675363286cf9f4381959e9edf6 Mon Sep 17 00:00:00 2001 From: Robert Kenny Date: Sat, 27 Jan 2024 10:29:58 +0000 Subject: [PATCH 3/4] bump circe optics version to avoid more dependency gronk --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index eb8c2794d2..d6c4853945 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -98,7 +98,7 @@ object ExternalDependencies { // This should match the version of circe used in scala-json; see // https://github.com/wellcomecollection/scala-json/blob/master/project/Dependencies.scala - val circeOptics = "0.13.0" + val circeOptics = "0.14.1" // This should match the version of aws used in scala-libs; see // https://github.com/wellcomecollection/scala-libs/blob/main/project/Dependencies.scala From 8a5da3589051db99a00faa1ba1ba42d6f925fa33 Mon Sep 17 00:00:00 2001 From: Robert Kenny Date: Tue, 6 Feb 2024 16:00:22 +0000 Subject: [PATCH 4/4] Adds a simple healthcheck endpoint to the bags and ingest apis (#1115) * Bump the SBT version from 1.4.1 to 1.9.8 * tedious transitive dependency fiddling * bump circe optics version to avoid more dependency gronk * Adds a simple healthcheck endpoint to the bags and ingest apis * update target groups to include HTTP healthchecks * update terraform module ref * Apply auto-formatting rules --------- Co-authored-by: Buildkite on behalf of Wellcome Collection --- .../storage_service/bags_api/BagsApi.scala | 166 ++++++++++-------- .../ingests_api/IngestsApi.scala | 25 ++- terraform/modules/service/bags/main.tf | 6 +- .../modules/service/bags/target_group.tf | 4 +- terraform/modules/service/bags/variables.tf | 5 + terraform/modules/service/base/main.tf | 8 +- terraform/modules/service/ingest/main.tf | 14 +- .../modules/service/ingest/target_group.tf | 4 +- terraform/modules/service/ingest/variables.tf | 5 + terraform/modules/service/worker/main.tf | 6 +- terraform/stack_prod/main.tf | 3 +- terraform/stack_staging/.terraform.lock.hcl | 19 +- terraform/stack_staging/main.tf | 3 +- 13 files changed, 157 insertions(+), 111 deletions(-) diff --git a/bags_api/src/main/scala/weco/storage_service/bags_api/BagsApi.scala b/bags_api/src/main/scala/weco/storage_service/bags_api/BagsApi.scala index b043f1db66..550bfe668c 100644 --- a/bags_api/src/main/scala/weco/storage_service/bags_api/BagsApi.scala +++ b/bags_api/src/main/scala/weco/storage_service/bags_api/BagsApi.scala @@ -22,16 +22,45 @@ trait BagsApi protected val s3PresignedUrls: S3PresignedUrls - private val routes: Route = pathPrefix("bags") { - concat( - // We look for /versions at the end of the path: this means we should - // return a list of versions, not the complete manifest. - // - // The trailing slash is significant: it causes Akka to consume "/versions", - // rather than "versions". If you omit it, the externalIdentifier gets a - // slash appended! - // - pathSuffix("versions" /) { + private val routes: Route = concat( + pathPrefix("bags") { + concat( + // We look for /versions at the end of the path: this means we should + // return a list of versions, not the complete manifest. + // + // The trailing slash is significant: it causes Akka to consume "/versions", + // rather than "versions". If you omit it, the externalIdentifier gets a + // slash appended! + // + pathSuffix("versions" /) { + path(Segment / Remaining) { + (space, remaining) => + get { + Try { decodeExternalIdentifier(remaining) } match { + case Success(externalIdentifier) => + val bagId = BagId( + space = StorageSpace(space), + externalIdentifier = externalIdentifier + ) + + parameter('before.as[String] ?) { maybeBefore => + withFuture { + lookupVersions( + bagId = bagId, + maybeBeforeString = maybeBefore + ) + } + } + + case Failure(_) => + notFound( + s"No storage manifest versions found for $space/$remaining" + ) + } + } + } + }, + // Look up a single manifest. path(Segment / Remaining) { (space, remaining) => get { @@ -42,84 +71,65 @@ trait BagsApi externalIdentifier = externalIdentifier ) - parameter('before.as[String] ?) { maybeBefore => - withFuture { - lookupVersions( - bagId = bagId, - maybeBeforeString = maybeBefore + val chemistAndDruggist = BagId( + space = StorageSpace("digitised"), + externalIdentifier = ExternalIdentifier("b19974760") + ) + + // This is some special casing to handle Chemist & Druggist, which + // is enormous. If somebody tries to retrieve it, direct them straight + // to the cached response. + // + // We should fix the bags API so retrieving this API doesn't cause the + // app to run out of heap space/memory. + // + // See https://github.com/wellcomecollection/platform/issues/4549 + bagId match { + case id if id == chemistAndDruggist => + val url = s3PresignedUrls + .getPresignedGetURL( + location = S3ObjectLocation( + bucket = + "wellcomecollection-storage-prod-large-response-cache", + key = "responses/digitised/b19974760/v1" + ), + expiryLength = 1 days + ) + .right + .get + + complete( + HttpResponse( + status = StatusCodes.TemporaryRedirect, + headers = Location(url.toExternalForm) :: Nil + ) ) - } + case _ => + parameter('version.as[String] ?) { maybeVersionString => + withFuture { + lookupBag( + bagId = bagId, + maybeVersionString = maybeVersionString + ) + } + } } case Failure(_) => - notFound( - s"No storage manifest versions found for $space/$remaining" - ) + notFound(s"Storage manifest $space/$remaining not found") } } } - }, - // Look up a single manifest. - path(Segment / Remaining) { (space, remaining) => + ) + }, + pathPrefix("management") { + path("healthcheck") { get { - Try { decodeExternalIdentifier(remaining) } match { - case Success(externalIdentifier) => - val bagId = BagId( - space = StorageSpace(space), - externalIdentifier = externalIdentifier - ) - - val chemistAndDruggist = BagId( - space = StorageSpace("digitised"), - externalIdentifier = ExternalIdentifier("b19974760") - ) - - // This is some special casing to handle Chemist & Druggist, which - // is enormous. If somebody tries to retrieve it, direct them straight - // to the cached response. - // - // We should fix the bags API so retrieving this API doesn't cause the - // app to run out of heap space/memory. - // - // See https://github.com/wellcomecollection/platform/issues/4549 - bagId match { - case id if id == chemistAndDruggist => - val url = s3PresignedUrls - .getPresignedGetURL( - location = S3ObjectLocation( - bucket = - "wellcomecollection-storage-prod-large-response-cache", - key = "responses/digitised/b19974760/v1" - ), - expiryLength = 1 days - ) - .right - .get - - complete( - HttpResponse( - status = StatusCodes.TemporaryRedirect, - headers = Location(url.toExternalForm) :: Nil - ) - ) - case _ => - parameter('version.as[String] ?) { maybeVersionString => - withFuture { - lookupBag( - bagId = bagId, - maybeVersionString = maybeVersionString - ) - } - } - } - - case Failure(_) => - notFound(s"Storage manifest $space/$remaining not found") - } + complete("ok") } } - ) - } + } + ) val bags: Route = wrapLargeResponses(routes) } diff --git a/ingests/ingests_api/src/main/scala/weco/storage_service/ingests_api/IngestsApi.scala b/ingests/ingests_api/src/main/scala/weco/storage_service/ingests_api/IngestsApi.scala index 28cf29d019..45a6fcdef3 100644 --- a/ingests/ingests_api/src/main/scala/weco/storage_service/ingests_api/IngestsApi.scala +++ b/ingests/ingests_api/src/main/scala/weco/storage_service/ingests_api/IngestsApi.scala @@ -10,15 +10,24 @@ trait IngestsApi[UnpackerDestination] extends CreateIngest[UnpackerDestination] with LookupIngest { - val ingests: Route = pathPrefix("ingests") { - post { - entity(as[RequestDisplayIngest]) { ingest => - withFuture { createIngest(ingest) } + val ingests: Route = concat( + pathPrefix("ingests") { + post { + entity(as[RequestDisplayIngest]) { ingest => + withFuture { createIngest(ingest) } + } + } ~ path(JavaUUID) { id: UUID => + get { + withFuture { lookupIngest(id) } + } } - } ~ path(JavaUUID) { id: UUID => - get { - withFuture { lookupIngest(id) } + }, + pathPrefix("management") { + path("healthcheck") { + get { + complete("ok") + } } } - } + ) } diff --git a/terraform/modules/service/bags/main.tf b/terraform/modules/service/bags/main.tf index 4338252363..eece143747 100644 --- a/terraform/modules/service/bags/main.tf +++ b/terraform/modules/service/bags/main.tf @@ -40,7 +40,7 @@ module "base" { } module "nginx_container" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/nginx/apigw?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/nginx/apigw?ref=v4.0.0" forward_port = var.container_port log_configuration = module.base.log_configuration @@ -51,7 +51,7 @@ module "nginx_container" { } module "app_container" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v4.0.0" name = "app" image = var.api_container_image @@ -62,7 +62,7 @@ module "app_container" { } module "tracker_container" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v4.0.0" name = "tracker" image = var.tracker_container_image diff --git a/terraform/modules/service/bags/target_group.tf b/terraform/modules/service/bags/target_group.tf index 4018bb8c28..baf30e91e1 100644 --- a/terraform/modules/service/bags/target_group.tf +++ b/terraform/modules/service/bags/target_group.tf @@ -14,7 +14,9 @@ resource "aws_lb_target_group" "tcp" { deregistration_delay = 90 health_check { - protocol = "TCP" + protocol = "HTTP" + path = var.healthcheck_path + matcher = "200" } } diff --git a/terraform/modules/service/bags/variables.tf b/terraform/modules/service/bags/variables.tf index f247f16f3b..3249fc2b1a 100644 --- a/terraform/modules/service/bags/variables.tf +++ b/terraform/modules/service/bags/variables.tf @@ -80,3 +80,8 @@ variable "nginx_container" { container_tag = string }) } + +variable "healthcheck_path" { + type = string + default = "/management/healthcheck" +} diff --git a/terraform/modules/service/base/main.tf b/terraform/modules/service/base/main.tf index bdaba81b6d..0dc481becb 100644 --- a/terraform/modules/service/base/main.tf +++ b/terraform/modules/service/base/main.tf @@ -1,5 +1,5 @@ module "log_router_container" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/firelens?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/firelens?ref=v4.0.0" namespace = var.service_name container_registry = var.logging_container["container_registry"] @@ -8,13 +8,13 @@ module "log_router_container" { } module "log_router_container_secrets_permissions" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v4.0.0" secrets = module.log_router_container.shared_secrets_logging role_name = module.task_definition.task_execution_role_name } module "task_definition" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/task_definition?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/task_definition?ref=v4.0.0" cpu = var.cpu memory = var.memory @@ -27,7 +27,7 @@ module "task_definition" { } module "service" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/service?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/service?ref=v4.0.0" cluster_arn = var.cluster_arn service_name = var.service_name diff --git a/terraform/modules/service/ingest/main.tf b/terraform/modules/service/ingest/main.tf index 62d4734192..d9875465c1 100644 --- a/terraform/modules/service/ingest/main.tf +++ b/terraform/modules/service/ingest/main.tf @@ -45,7 +45,7 @@ module "base" { } module "nginx_container" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/nginx/apigw?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/nginx/apigw?ref=v4.0.0" forward_port = var.external_api_container_port log_configuration = module.base.log_configuration @@ -56,7 +56,7 @@ module "nginx_container" { } module "external_api_container" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v4.0.0" name = "api" image = var.external_api_container_image @@ -68,13 +68,13 @@ module "external_api_container" { } module "external_api_container_secrets_permissions" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v4.0.0" secrets = var.external_api_secrets role_name = module.base.task_execution_role_name } module "worker_container" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v4.0.0" name = "worker" image = var.worker_container_image @@ -86,13 +86,13 @@ module "worker_container" { } module "worker_container_secrets_permissions" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v4.0.0" secrets = var.worker_secrets role_name = module.base.task_execution_role_name } module "internal_api_container" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v4.0.0" name = "tracker" image = var.internal_api_container_image @@ -104,7 +104,7 @@ module "internal_api_container" { } module "internal_api_container_secrets_permissions" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v4.0.0" secrets = var.internal_api_secrets role_name = module.base.task_execution_role_name } diff --git a/terraform/modules/service/ingest/target_group.tf b/terraform/modules/service/ingest/target_group.tf index 4018bb8c28..baf30e91e1 100644 --- a/terraform/modules/service/ingest/target_group.tf +++ b/terraform/modules/service/ingest/target_group.tf @@ -14,7 +14,9 @@ resource "aws_lb_target_group" "tcp" { deregistration_delay = 90 health_check { - protocol = "TCP" + protocol = "HTTP" + path = var.healthcheck_path + matcher = "200" } } diff --git a/terraform/modules/service/ingest/variables.tf b/terraform/modules/service/ingest/variables.tf index 1bd5216d7c..991ef1ca79 100644 --- a/terraform/modules/service/ingest/variables.tf +++ b/terraform/modules/service/ingest/variables.tf @@ -111,3 +111,8 @@ variable "nginx_container" { container_tag = string }) } + +variable "healthcheck_path" { + type = string + default = "/management/healthcheck" +} \ No newline at end of file diff --git a/terraform/modules/service/worker/main.tf b/terraform/modules/service/worker/main.tf index 3622eaebe7..9975a65069 100644 --- a/terraform/modules/service/worker/main.tf +++ b/terraform/modules/service/worker/main.tf @@ -24,7 +24,7 @@ module "base" { } module "app_container" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v4.0.0" name = "app" image = var.container_image @@ -36,13 +36,13 @@ module "app_container" { } module "app_container_secrets_permissions" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v4.0.0" secrets = var.secrets role_name = module.base.task_execution_role_name } module "scaling" { - source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/autoscaling?ref=v3.13.1" + source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/autoscaling?ref=v4.0.0" name = var.service_name diff --git a/terraform/stack_prod/main.tf b/terraform/stack_prod/main.tf index e5f1b92a1c..01f7ffdaf8 100644 --- a/terraform/stack_prod/main.tf +++ b/terraform/stack_prod/main.tf @@ -86,8 +86,7 @@ module "stack_prod" { logging_container = { container_registry = "760097843905.dkr.ecr.eu-west-1.amazonaws.com/uk.ac.wellcome" container_name = "fluentbit" - # https://github.com/wellcomecollection/terraform-aws-ecs-service/tree/v3.13.1 - container_tag = "ac9e48d11d76f3d3abd64e4a9440462539da2e7a" + container_tag = "ac9e48d11d76f3d3abd64e4a9440462539da2e7a" } nginx_container = { diff --git a/terraform/stack_staging/.terraform.lock.hcl b/terraform/stack_staging/.terraform.lock.hcl index 6a9a480a18..ae321ee2bd 100644 --- a/terraform/stack_staging/.terraform.lock.hcl +++ b/terraform/stack_staging/.terraform.lock.hcl @@ -2,8 +2,23 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "5.13.0" + version = "5.35.0" hashes = [ - "h1:Ui8l44bQHjYYoEHQTPBjtI7qvCzPoHLaHZL8yA/gAJk=", + "h1:MKNFmhsOIirK7Qzr6TWkVaBcVGN81lCU0BPiaPOeQ8s=", + "zh:3a2a6f40db82d30ea8c5e3e251ca5e16b08e520570336e7e342be823df67e945", + "zh:420a23b69b412438a15b8b2e2c9aac2cf2e4976f990f117e4bf8f630692d3949", + "zh:4d8b887f6a71b38cff77ad14af9279528433e279eed702d96b81ea48e16e779c", + "zh:4edd41f8e1c7d29931608a7b01a7ae3d89d6f95ef5502cf8200f228a27917c40", + "zh:6337544e2ded5cf37b55a70aa6ce81c07fd444a2644ff3c5aad1d34680051bdc", + "zh:668faa3faaf2e0758bf319ea40d2304340f4a2dc2cd24460ddfa6ab66f71b802", + "zh:79ddc6d7c90e59fdf4a51e6ea822ba9495b1873d6a9d70daf2eeaf6fc4eb6ff3", + "zh:885822027faf1aa57787f980ead7c26e7d0e55b4040d926b65709b764f804513", + "zh:8c50a8f397b871388ff2e048f5eb280af107faa2e8926694f1ffd9f32a7a7cdf", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a2f5d2553df5573a060641f18ee7585587047c25ba73fd80617f59b5893d22b4", + "zh:c43833ae2a152213ee92eb5be7653f9493779eddbe0ce403ea49b5f1d87fd766", + "zh:dab01527a3a55b4f0f958af6f46313d775e27f9ad9d10bedbbfea4a35a06dc5f", + "zh:ed49c65620ec42718d681a7fc00c166c295ff2795db6cede2c690b83f9fb3e65", + "zh:f0a358c0ae1087c466d0fbcc3b4da886f33f881a145c3836ec43149878b86a1a", ] } diff --git a/terraform/stack_staging/main.tf b/terraform/stack_staging/main.tf index bec6a15287..a672364b2f 100644 --- a/terraform/stack_staging/main.tf +++ b/terraform/stack_staging/main.tf @@ -91,8 +91,7 @@ module "stack_staging" { logging_container = { container_registry = "760097843905.dkr.ecr.eu-west-1.amazonaws.com/uk.ac.wellcome" container_name = "fluentbit" - # https://github.com/wellcomecollection/terraform-aws-ecs-service/tree/v3.13.1 - container_tag = "ac9e48d11d76f3d3abd64e4a9440462539da2e7a" + container_tag = "ac9e48d11d76f3d3abd64e4a9440462539da2e7a" } nginx_container = {