Merge branch 'develop' into jgd_WX-1122_appinsights

.github/workflows/consumer_contract_tests.yml

@@ -0,0 +1,145 @@
+name: Consumer contract tests
+# The purpose of this workflow is to run a suite of Cromwell contract tests against mock service provider(s) using Pact framework.
+#
+# More details about Contract Testing can be found in our handbook
+#
+# https://broadworkbench.atlassian.net/wiki/spaces/IRT/pages/2660368406/Getting+Started+with+Pact+Contract+Testing
+#
+# This workflow involves Cromwell as a consumer, and ANY provider (e.g. Sam) Cromwell consumes.
+# Each party owns a set of tests (aka contract tests).
+#
+#   Consumer contract tests (aka consumer tests) runs on a mock provider service and does not require a real provider service.
+#   Provider contract tests (aka provider verification tests) runs independently of any consumer.
+#
+# Specifically:
+#   Cromwell runs consumer tests against mock service. Upon success, publish consumer pacts to
+#   Pact Broker https://pact-broker.dsp-eng-tools.broadinstitute.org/.
+#
+#   Pact Broker is the source of truth to forge contractual obligations between consumer and provider.
+#
+#   This workflow meets the criteria of Pact Broker *Platinum* as described in https://docs.pact.io/pact_nirvana/step_6.
+#   The can-i-deploy job has been added to this workflow to support *Platinum* and gate the code for promotion to default branch.
+#
+#   This is how it works.
+#
+#     Consumer makes a change that results in a new pact published to Pact Broker.
+#     Pact Broker notifies provider(s) of the changed pact and trigger corresponding verification workflows.
+#     Provider downloads relevant versions of consumer pacts from Pact Broker and kicks off verification tests against the consumer pacts.
+#     Provider updates Pact Broker with verification status.
+#     Consumer kicks off can-i-deploy on process to determine if changes can be promoted and used for deployment.
+#
+# NOTE: The publish-contracts workflow will use the latest commit of the branch that triggers this workflow to publish the unique consumer contract version to Pact Broker.
+
+on:
+  pull_request:
+    branches:
+      - develop
+    paths-ignore:
+      - 'README.md'
+  push:
+    branches:
+      - develop
+    paths-ignore:
+      - 'README.md'
+  merge_group:
+    branches:
+      - develop
+
+jobs:
+  init-github-context:
+    runs-on: ubuntu-latest
+    outputs:
+      repo-branch: ${{ steps.extract-branch.outputs.repo-branch }}
+      repo-version: ${{ steps.extract-branch.outputs.repo-version }}
+      fork: ${{ steps.extract-branch.outputs.fork }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Obtain branch properties
+        id: extract-branch
+        run: |
+          FORK=false
+          GITHUB_EVENT_NAME=${{ github.event_name }}
+          if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then
+            GITHUB_REF=${{ github.ref }}
+            GITHUB_SHA=${{ github.sha }}
+          elif [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
+            FORK=${{ github.event.pull_request.head.repo.fork }}
+            GITHUB_REF=refs/heads/${{ github.head_ref }}
+            GITHUB_SHA=${{ github.event.pull_request.head.sha }}
+          elif [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then
+            GITHUB_REF=refs/heads/${{ github.head_ref }}
+          else
+            echo "Failed to extract branch information"
+            exit 1
+          fi
+          echo "repo-branch=${GITHUB_REF/refs\/heads\//""}" >> $GITHUB_OUTPUT
+          echo "repo-version=${GITHUB_SHA}" >> $GITHUB_OUTPUT
+          echo "fork=${FORK}" >> $GITHUB_OUTPUT
+
+      - name: Is PR triggered by forked repo?
+        if: ${{ steps.extract-branch.outputs.fork == 'true' }}
+        run: |
+          echo "PR was triggered by forked repo"
+
+      - name: Echo repo and branch information
+        run: |
+          echo "repo-owner=${{ github.repository_owner }}"
+          echo "repo-name=${{ github.event.repository.name }}"
+          echo "repo-branch=${{ steps.extract-branch.outputs.repo-branch }}"
+          echo "repo-version=${{ steps.extract-branch.outputs.repo-version }}"
+
+  cromwell-consumer-contract-tests:
+    runs-on: ubuntu-latest
+    needs: [init-github-context]
+    outputs:
+      pact-b64: ${{ steps.encode-pact.outputs.pact-b64 }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run consumer tests
+        run: |
+          docker run --rm -v $PWD:/working \
+                          -v jar-cache:/root/.ivy \
+                          -v jar-cache:/root/.ivy2 \
+                          -w /working \
+                          sbtscala/scala-sbt:openjdk-17.0.2_1.7.2_2.13.10 \
+                          sbt "project pact4s" clean test
+
+      - name: Output consumer contract as non-breaking base64 string
+        id: encode-pact
+        run: |
+          cd pact4s
+          NON_BREAKING_B64=$(cat target/pacts/cromwell-consumer-drshub-provider.json | base64 -w 0)
+          echo "pact-b64=${NON_BREAKING_B64}" >> $GITHUB_OUTPUT
+
+  # Prevent untrusted sources from using PRs to publish contracts
+  # since access to secrets is not allowed.
+  publish-contracts:
+    runs-on: ubuntu-latest
+    if: ${{ needs.init-github-context.outputs.fork == 'false' }}
+    needs: [init-github-context, cromwell-consumer-contract-tests]
+    steps:
+      - name: Dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: .github/workflows/publish-contracts.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # github token for access to kick off a job in the private repo
+          inputs: '{ "pact-b64": "${{ needs.cromwell-consumer-contract-tests.outputs.pact-b64 }}", "repo-owner": "${{ github.repository_owner }}", "repo-name": "${{ github.event.repository.name }}", "repo-branch": "${{ needs.init-github-context.outputs.repo-branch }}" }'
+
+  can-i-deploy:
+    runs-on: ubuntu-latest
+    if: ${{ needs.init-github-context.outputs.fork == 'false' && false}} # Disabling this step for now until there is a webhook setup
+    needs: [ init-github-context, publish-contracts ]
+    steps:
+      - name: Dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: .github/workflows/can-i-deploy.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # github token for access to kick off a job in the private repo
+          inputs: '{ "pacticipant": "cromwell-consumer", "version": "${{ needs.init-github-context.outputs.repo-version }}" }'

build.sbt

@@ -354,6 +354,11 @@ lazy val `cromwell-drs-localizer` = project
   .dependsOn(common)
   .dependsOn(`cloud-nio-impl-drs` % "test->test")
 
+lazy val pact4s = project.in(file("pact4s"))
+  .settings(pact4sSettings)
+  .dependsOn(services)
+  .disablePlugins(sbtassembly.AssemblyPlugin)
+
 lazy val server = project
   .withExecutableSettings("cromwell", serverDependencies)
   .dependsOn(engine)
@@ -419,4 +424,5 @@ lazy val root = (project in file("."))
   .aggregate(wes2cromwell)
   .aggregate(wom)
   .aggregate(womtool)
+  .aggregate(pact4s)
   .withAggregateSettings()

dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2Abstract.scala

@@ -70,8 +70,9 @@ object DockerRegistryV2Abstract {
   }
 
   // Placeholder exceptions that can be carried through IO before being converted to a DockerInfoFailedResponse
-  private class Unauthorized() extends Exception
-  private class NotFound() extends Exception
+  private class Unauthorized(message: String) extends Exception(message)
+  private class NotFound(message: String) extends Exception(message)
+  private class UnknownError(message: String) extends Exception(message)
 }
 
 /**
@@ -131,7 +132,10 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi
   protected def getDockerResponse(token: Option[String], dockerInfoContext: DockerInfoContext)(implicit client: Client[IO]): IO[DockerInfoSuccessResponse] = {
     val requestDockerManifest = manifestRequest(token, dockerInfoContext.dockerImageID, AcceptDockerManifestV2Header)
     lazy val requestOCIManifest = manifestRequest(token, dockerInfoContext.dockerImageID, AcceptOCIIndexV1Header)
-    def tryOCIManifest(err: Throwable) = executeRequest(requestOCIManifest, handleManifestResponse(dockerInfoContext, token))
+    def tryOCIManifest(err: Throwable) = {
+      logger.info(s"Manifest request failed for docker manifest V2, falling back to OCI manifest. Image: ${dockerInfoContext.dockerImageID}", err)
+      executeRequest(requestOCIManifest, handleManifestResponse(dockerInfoContext, token))
+    }
     // Try to execute a request using the Docker Manifest format, and if that fails, try using the newer OCI manifest format
     executeRequest(requestDockerManifest, handleManifestResponse(dockerInfoContext, token))
       .handleErrorWith(tryOCIManifest)
@@ -282,9 +286,9 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi
 
   private def getDigestFromResponse(response: Response[IO]): IO[DockerHashResult] = response match {
     case Status.Successful(r) => extractDigestFromHeaders(r.headers)
-    case Status.Unauthorized(_) => IO.raiseError(new Unauthorized)
-    case Status.NotFound(_) => IO.raiseError(new NotFound)
-    case failed => failed.as[String].flatMap(body => IO.raiseError(new Exception(s"Failed to get manifest: $body"))
+    case Status.Unauthorized(r) => r.as[String].flatMap(body => IO.raiseError(new Unauthorized(r.status.toString + " " + body)))
+    case Status.NotFound(r) => r.as[String].flatMap(body => IO.raiseError(new NotFound(r.status.toString + " " + body)))
+    case failed => failed.as[String].flatMap(body => IO.raiseError(new UnknownError(failed.status.toString + " " + body))
     )
   }
 

pact4s/README.md

@@ -0,0 +1,51 @@
+c# pact4s [Under construction]
+
+pact4s is used for contract testing.
+
+# Dependencies
+
+```scala
+  val pact4sDependencies = Seq(
+    pact4sScalaTest,
+    pact4sCirce,
+    http4sEmberClient,
+    http4sDsl,
+    http4sEmberServer,
+    http4sCirce,
+    circeCore,
+    typelevelCat,
+    scalaTest
+  )
+
+lazy val pact4s = project.in(file("pact4s"))
+  .settings(pact4sSettings)
+  .dependsOn(http % "test->test;compile->compile")
+```
+
+## Building and running contract tests
+Clone the repo.
+```
+$ git clone https://github.com/broadinstitute/cromwell.git 
+$ cd cromwell
+```
+
+If you are already using OpenJDK 11, run the following command. 
+```
+$ sbt "project pact4s" clean test  
+```
+
+Otherwise, you can run the command inside a docker container with OpenJDK 11 installed. 
+This is especially useful when automating contract tests in a GitHub Action runner which does not guarantee the correct OpenJDK version.
+```
+docker run --rm -v $PWD:/working \
+                -v jar-cache:/root/.ivy \
+                -v jar-cache:/root/.ivy2 \
+                -w /working \
+                sbtscala/scala-sbt:openjdk-11.0.16_1.8.1_2.13.10 \
+                sbt "project pact4s" clean test
+```
+
+The generated contracts can be found in the `./target/pacts` folder
+- `cromwell-consumer-drshub-provider.json`
+- `cromwell-consumer-fake-provider.json`
+

pact4s/src/main/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/DrsHubClient.scala

@@ -0,0 +1,93 @@
+package org.broadinstitute.dsde.workbench.cromwell.consumer
+
+import cats.effect.Concurrent
+import cats.syntax.all._
+import com.typesafe.scalalogging.LazyLogging
+import io.circe.{Decoder, Encoder}
+import org.http4s._
+import org.http4s.circe.CirceEntityCodec.{circeEntityDecoder, circeEntityEncoder}
+import org.http4s.client.Client
+
+case class AccessUrl(url: String, headers: List[String])
+
+case class ResourceMetadataRequest(url: String, fields: List[String])
+
+case class ResourceMetadata(contentType: String,
+                            size: Long,
+                            timeCreated: String,
+                            timeUpdated: String,
+                            bucket: Option[String],
+                            name: Option[String],
+                            gsUri: Option[String],
+                            googleServiceAccount: Option[Map[String, Map[String, String]]],
+                            fileName: Option[String],
+                            accessUrl: Option[AccessUrl],
+                            hashes: Map[String, String],
+                            localizationPath: Option[String],
+                            bondProvider: Option[String]
+)
+trait DrsHubClient[F[_]] extends LazyLogging {
+  def fetchSystemStatus(): F[Boolean]
+
+  def resolveDrsObject(drsPath: String, fields: List[String]): F[ResourceMetadata]
+
+}
+
+/*
+ This class represents the consumer (Cromwell) view of the DrsHub provider that implements the following endpoints:
+  - GET /status
+  - GET /api/v4/drs/resolve
+ */
+class DrsHubClientImpl[F[_]: Concurrent](client: Client[F], baseUrl: Uri) extends DrsHubClient[F] {
+  val apiVersion = "v4"
+
+  implicit val accessUrlDecoder: Decoder[AccessUrl] = Decoder.forProduct2("url", "headers")(AccessUrl.apply)
+  implicit val resourceMetadataDecoder: Decoder[ResourceMetadata] = Decoder.forProduct13(
+    "contentType",
+    "size",
+    "timeCreated",
+    "timeUpdated",
+    "bucket",
+    "name",
+    "gsUri",
+    "googleServiceAccount",
+    "fileName",
+    "accessUrl",
+    "hashes",
+    "localizationPath",
+    "bondProvider"
+  )(ResourceMetadata.apply)
+  implicit val resourceMetadataRequestEncoder: Encoder[ResourceMetadataRequest] = Encoder.forProduct2("url", "fields")(x =>
+    (x.url, x.fields)
+  )
+  implicit val resourceMetadataRequestEntityEncoder: EntityEncoder[F, ResourceMetadataRequest] = circeEntityEncoder[F, ResourceMetadataRequest]
+  override def fetchSystemStatus(): F[Boolean] = {
+    val request = Request[F](uri = baseUrl / "status").withHeaders(
+      org.http4s.headers.Accept(MediaType.application.json)
+    )
+    client.run(request).use { resp =>
+      resp.status match {
+        case Status.Ok                  => true.pure[F]
+        case Status.InternalServerError => false.pure[F]
+        case _                          => UnknownError.raiseError
+      }
+    }
+  }
+
+  override def resolveDrsObject(drsPath: String, fields: List[String]): F[ResourceMetadata] = {
+    val body = ResourceMetadataRequest(url = drsPath, fields = fields)
+    val entityBody: EntityBody[F] = EntityEncoder[F, ResourceMetadataRequest].toEntity(body).body
+    val request = Request[F](uri = baseUrl / "api" / apiVersion / "drs" / "resolve", method=Method.POST, body=entityBody).withHeaders(
+      org.http4s.headers.`Content-Type`(MediaType.application.json)
+    )
+    client.run(request).use { resp =>
+      resp.status match {
+        case Status.Ok                  => resp.as[ResourceMetadata]
+        case _                          => UnknownError.raiseError
+      }
+    }
+  }
+
+}
+
+