[WX-1410] Sanitize 4 byte UTF-8 characters before inserting into METADATA_ENTRY

core/src/main/resources/reference.conf

@@ -540,6 +540,7 @@ services {
     config {
       # See cromwell.examples.conf for details on settings one can use here as they depend on the implementation
       # being used.
+      metadata-keys-to-sanitize-utf8mb4: []
     }
   }
   Instrumentation {

core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala

@@ -37,4 +37,6 @@ object WorkflowMetadataKeys {
   val Labels = "labels"
   val MetadataArchiveStatus = "metadataArchiveStatus"
   val Message = "message"
+
+  val CommandLine = "commandLine"
 }

cromwell.example.backends/cromwell.examples.conf

@@ -498,6 +498,11 @@ services {
     #   # count against this limit.
     #   metadata-read-row-number-safety-threshold = 1000000
     #
+    #   # Remove any UTF-8 mb4 (4 byte) characters from metadata keys in the list.
+    #   # These characters (namely emojis) will cause metadata writing to fail in database collations
+    #   # that do not support 4 byte UTF-8 characters.
+    #   metadata-keys-to-sanitize-utf8mb4 = ["submittedFiles:workflow", "commandLine"]
+    #
     #  metadata-write-statistics {
     #    # Not strictly necessary since the 'metadata-write-statistics' section itself is enough for statistics to be recorded.
     #    # However, this can be set to 'false' to disable statistics collection without deleting the section.

services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala

@@ -2,14 +2,17 @@
 
 import akka.actor.{ActorLogging, ActorRef, Props}
 import cats.data.NonEmptyVector
+import com.typesafe.config.ConfigFactory
 import cromwell.core.Dispatcher.ServiceDispatcher
 import cromwell.core.Mailbox.PriorityMailbox
 import cromwell.core.WorkflowId
 import cromwell.core.instrumentation.InstrumentationPrefixes
-import cromwell.services.metadata.MetadataEvent
+import cromwell.services.metadata.{MetadataEvent, MetadataValue}
 import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata.impl.MetadataStatisticsRecorder.MetadataStatisticsRecorderSettings
 import cromwell.services.{EnhancedBatchActor, MetadataServicesStore}
+import net.ceedubs.ficus.Ficus._
+import wdl.util.StringUtil
 
 import scala.concurrent.duration._
 import scala.util.{Failure, Success}
@@ -27,9 +30,14 @@
   private val statsRecorder = MetadataStatisticsRecorder(metadataStatisticsRecorderSettings)
 
   override def process(e: NonEmptyVector[MetadataWriteAction]) = instrumentedProcess {
+    val config = ConfigFactory.load().getConfig("services.MetadataService.config")
+    val metadataKeysToCleanOption = config.as[Option[List[String]]]("metadata-keys-to-sanitize-utf8mb4")
+    val metadataKeysToClean = metadataKeysToCleanOption.getOrElse(List())
+
+    val cleanedMetadataWriteActions = if (metadataKeysToClean.isEmpty) e else sanitizeInputs(e, metadataKeysToClean)
     val empty = (Vector.empty[MetadataEvent], List.empty[(Iterable[MetadataEvent], ActorRef)])
 
-    val (putWithoutResponse, putWithResponse) = e.foldLeft(empty) {
+    val (putWithoutResponse, putWithResponse) = cleanedMetadataWriteActions.foldLeft(empty) {
       case ((putEvents, putAndRespondEvents), action: PutMetadataAction) =>
         (putEvents ++ action.events, putAndRespondEvents)
       case ((putEvents, putAndRespondEvents), action: PutMetadataActionAndRespond) =>
@@ -46,7 +54,7 @@
       case Success(_) =>
         putWithResponse foreach { case (ev, replyTo) => replyTo ! MetadataWriteSuccess(ev) }
       case Failure(cause) =>
-        val (outOfTries, stillGood) = e.toVector.partition(_.maxAttempts <= 1)
+        val (outOfTries, stillGood) = cleanedMetadataWriteActions.toVector.partition(_.maxAttempts <= 1)
 
         handleOutOfTries(outOfTries, cause)
         handleEventsToReconsider(stillGood)
@@ -55,6 +63,43 @@
     dbAction.map(_ => allPutEvents.size)
   }
 
+  private def sanitizeInputs(
+    metadataWriteActions: NonEmptyVector[MetadataWriteAction],
+    metadataKeysToClean: List[String]
+  ): NonEmptyVector[MetadataWriteAction] =
+    metadataWriteActions.map { metadataWriteAction =>
+      val metadataEvents =
+        metadataWriteAction.events.map { event =>
+          if (
+            metadataKeysToClean.contains(event.key.key)
+            || (event.key.key.contains(":")
+              && metadataKeysToClean.contains(event.key.key.substring(0, event.key.key.indexOf(":"))))
+          ) {
+            event.value match {
+              case Some(_) =>
+                val value = event.value.get
+                MetadataEvent(event.key.copy(),
+                              Option(MetadataValue(StringUtil.cleanUtf8mb4(value.value), value.valueType)),
+                              event.offsetDateTime
+                )
+              case _ => event
+            }
+          } else {
+            event
+          }
+        }
+      metadataWriteAction match {
+        case action: PutMetadataAction =>
+          PutMetadataAction(metadataEvents, action.maxAttempts)
+        case action: PutMetadataActionAndRespond =>
+          PutMetadataActionAndRespond(
+            metadataEvents,
+            action.replyTo,
+            action.maxAttempts
+          )
+      }
+    }
+
   private def countActionsByWorkflow(writeActions: Vector[MetadataWriteAction]): Map[WorkflowId, Int] =
     writeActions.flatMap(_.events).groupBy(_.key.workflowId).map { case (k, v) => k -> v.size }
 

supportedBackends/google/pipelines/v2beta/src/main/scala/cromwell/backend/google/pipelines/v2beta/PipelinesUtilityConversions.scala

@@ -8,6 +8,7 @@ import cromwell.core.ExecutionEvent
 import cromwell.core.logging.JobLogger
 import mouse.all._
 import PipelinesUtilityConversions._
+import wdl.util.StringUtil
 
 import scala.language.postfixOps
 
@@ -67,7 +68,7 @@ trait PipelinesUtilityConversions {
     // characters (typically emoji). Some databases have trouble storing these; replace them with the standard
     // "unknown character" unicode symbol.
     val name = Option(event.getContainerStopped) match {
-      case Some(_) => cleanUtf8mb4(event.getDescription)
+      case Some(_) => StringUtil.cleanUtf8mb4(event.getDescription)
       case _ => event.getDescription
     }
 
@@ -101,9 +102,4 @@ object PipelinesUtilityConversions {
         None
       }
   }
-
-  lazy val utf8mb4Regex = "[\\x{10000}-\\x{FFFFF}]"
-  lazy val utf8mb3Replacement = "\uFFFD" // This is the standard char for replacing invalid/unknown unicode chars
-  def cleanUtf8mb4(in: String): String =
-    in.replaceAll(utf8mb4Regex, utf8mb3Replacement)
 }

wdl/model/draft2/src/test/scala/wdl/util/StringUtilSpec.scala

@@ -70,4 +70,21 @@ class StringUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers
     }
   }
 
+  it should "not modify strings that contain only ascii characters" in {
+    val input = "hi there!?"
+    StringUtil.cleanUtf8mb4(input) shouldBe input
+  }
+
+  it should "not modify strings with 3-byte unicode characters" in {
+    val input = "Here is my non-ascii character: \u1234 Do you like it?"
+    StringUtil.cleanUtf8mb4(input) shouldBe input
+  }
+
+  it should "replace 4-byte unicode characters" in {
+    val cry = new String(Character.toChars(Integer.parseInt("1F62D", 16)))
+    val barf = new String(Character.toChars(Integer.parseInt("1F92E", 16)))
+    val input = s"When I try to put an emoji in the database it $barf and then I $cry"
+    val cleaned = "When I try to put an emoji in the database it \uFFFD and then I \uFFFD"
+    StringUtil.cleanUtf8mb4(input) shouldBe cleaned
+  }
 }

wom/src/main/scala/wdl/util/StringUtil.scala

@@ -8,6 +8,8 @@ import scala.annotation.tailrec
   * WOMmy TaskDefinition. That should get straightened out. */
 object StringUtil {
   val Ws = Pattern.compile("[\\ \\t]+")
+  val utf8mb4Regex = "[\\x{10000}-\\x{FFFFF}]"
+  val utf8mb3Replacement = "\uFFFD" // This is the standard char for replacing
 
   /**
     * 1) Remove all leading newline chars
@@ -63,4 +65,12 @@ object StringUtil {
 
     start(0)
   }
+
+  /**
+   * Remove all utf8mb4 exclusive characters (emoji) from the given string.
+   * @param in String to clean
+   * @return String with all utf8mb4 exclusive characters removed
+   */
+  def cleanUtf8mb4(in: String): String =
+    in.replaceAll(utf8mb4Regex, utf8mb3Replacement)
 }