WX-1153 Azure blob read md5 from metadata for large files

filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilder.scala

@@ -7,6 +7,7 @@
 
 import java.net.{MalformedURLException, URI}
 import java.nio.file.{Files, LinkOption}
+import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
@@ -78,6 +79,17 @@
   //    format the library expects
   // 2) If the path looks like <container>:<path>, strip off the <container>: to leave the absolute path inside the container.
   private val brokenPathRegex = "https:/([a-z0-9]+).blob.core.windows.net/([-a-zA-Z0-9]+)/(.*)".r
+
+  // Blob files larger than 5 GB upload in parallel parts [0][1] and do not get a native `CONTENT-MD5` property.
+  // Instead, some uploaders such as TES [2] may optionally calculate the MD5 themselves and store it under this key in metadata.
+  // N.B. most if not virtually all large files in the wild will NOT have this key populated because they were not created by TES. [3]
+  //
+  // [0] https://learn.microsoft.com/en-us/azure/storage/blobs/scalability-targets
+  // [1] https://learn.microsoft.com/en-us/rest/api/storageservices/version-2019-12-12
+  // [2] https://github.com/microsoft/ga4gh-tes/pull/236
+  // [3] As of 2023-08 there are zero search engine results for `md5_hashlist_root_hash` and the only sure-thing client is TES
+  private val largeBlobFileMetadataKey = "md5_hashlist_root_hash"
+
   def cleanedNioPathString(nioString: String): String = {
     val pathStr = nioString match {
       case brokenPathRegex(_, containerName, pathInContainer) =>
@@ -116,16 +128,32 @@
   def blobFileAttributes: Try[AzureBlobFileAttributes] =
     Try(Files.readAttributes(nioPath, classOf[AzureBlobFileAttributes]))
 
+  def blobFileMetadata: Try[Option[Map[String, String]]] = blobFileAttributes.map { attrs =>
+    // `metadata()` has a documented `null` case
+    Option(attrs.metadata()).map(_.asScala.toMap)
+  }
+
   def md5HexString: Try[Option[String]] = {
-    blobFileAttributes.map(h =>
-      Option(h.blobHttpHeaders().getContentMd5) match {
-        case None => None
-        case Some(arr) if arr.isEmpty => None
-        // Convert the bytes to a hex-encoded string. Note that this value
-        // is rendered in base64 in the Azure web portal.
-        case Some(bytes) => Option(bytes.map("%02x".format(_)).mkString)
+    def md5FromMetadata: Option[String] = (blobFileMetadata map { maybeMetadataMap: Option[Map[String, String]] =>
+      maybeMetadataMap flatMap { metadataMap: Map[String, String] =>
+        metadataMap.get(BlobPath.largeBlobFileMetadataKey)
+      }
+    }).toOption.flatten
+
+    // Convert the bytes to a hex-encoded string. Note that the value
+    // is rendered in base64 in the Azure web portal.
+    def hexString(bytes: Array[Byte]): String = bytes.map("%02x".format(_)).mkString
+
+    blobFileAttributes.map { attr: AzureBlobFileAttributes =>
+      (Option(attr.blobHttpHeaders().getContentMd5), md5FromMetadata) match {
+        case (None, None) => None
+        case (None, Some(metadataMd5)) => Option(metadataMd5)
+        case (Some(headerMd5Bytes), None) if headerMd5Bytes.isEmpty => None
+        // (Some, Some) could happen if an uploader redundantly populates an md5 for a small file.
+        // Doesn't seem like an erroneous condition so just choose the native one.
+        case (Some(headerMd5Bytes), _) => Option(hexString(headerMd5Bytes))
       }
-    )
+    }
   }
 
   /**

filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderSpec.scala

@@ -100,6 +100,30 @@ class BlobPathBuilderSpec extends AnyFlatSpec with Matchers with MockSugar {
     new BlobPathBuilder(store, endpoint)(fsm)
   }
 
+  it should "read md5 from small files <5g" in {
+    val builder = makeBlobPathBuilder(endpoint, store)
+    val evalPath = "/test/inputFile.txt"
+    val testString = endpoint.value + "/" + store + evalPath
+    val blobPath1: BlobPath = builder build testString getOrElse fail()
+    blobPath1.md5HexString.toOption.get should equal(Some("967f0f086992f1a8b48f0a533f80290b"))
+  }
+
+  it should "read md5 from large files >5g" in {
+    val builder = makeBlobPathBuilder(endpoint, store)
+    val evalPath = "/test/Rocky-9.2-aarch64-dvd.iso"
+    val testString = endpoint.value + "/" + store + evalPath
+    val blobPath1: BlobPath = builder build testString getOrElse fail()
+    blobPath1.md5HexString.toOption.get should equal(Some("13cb09331d2d12c0f476f81c672a4319"))
+  }
+
+  it should "choose the native md5 over the metadata md5 for files that have both" in {
+    val builder = makeBlobPathBuilder(endpoint, store)
+    val evalPath = "/test/redundant_md5_test.txt"
+    val testString = endpoint.value + "/" + store + evalPath
+    val blobPath1: BlobPath = builder build testString getOrElse fail()
+    blobPath1.md5HexString.toOption.get should equal(Some("9e5ceec07c8730b593a3a4b4ae324475"))
+  }
+
   ignore should "resolve an absolute path string correctly to a path" in {
     val builder = makeBlobPathBuilder(endpoint, store)
     val rootString = s"${endpoint.value}/${store.value}/cromwell-execution"