Fix EtlPipeline short-circuiting on a first bad row (close snowplow/s…

…nowplow#4320)

src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlPipeline.scala

@@ -13,20 +13,27 @@
 package com.snowplowanalytics.snowplow.enrich.common
 
 import cats.Monad
-import cats.data.{EitherT, NonEmptyList, Validated, ValidatedNel}
+import cats.data.{Validated, ValidatedNel}
 import cats.effect.Clock
 import cats.implicits._
+
+import io.circe.Json
+
 import com.snowplowanalytics.iglu.client.Client
 import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup
-import com.snowplowanalytics.snowplow.badrows._
-import io.circe.Json
+
+import com.snowplowanalytics.snowplow.badrows.{BadRow, Processor}
+
 import org.joda.time.DateTime
 
-import adapters.AdapterRegistry
-import enrichments.{EnrichmentManager, EnrichmentRegistry}
-import loaders.CollectorPayload
-import outputs.EnrichedEvent
-import utils.HttpClient
+import com.snowplowanalytics.snowplow.enrich.common.adapters.AdapterRegistry
+import com.snowplowanalytics.snowplow.enrich.common.enrichments.{
+  EnrichmentManager,
+  EnrichmentRegistry
+}
+import com.snowplowanalytics.snowplow.enrich.common.loaders.CollectorPayload
+import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent
+import com.snowplowanalytics.snowplow.enrich.common.utils.HttpClient
 
 /** Expresses the end-to-end event pipeline supported by the Scala Common Enrich project. */
 object EtlPipeline {
@@ -52,37 +59,30 @@ object EtlPipeline {
     processor: Processor,
     etlTstamp: DateTime,
     input: ValidatedNel[BadRow, Option[CollectorPayload]]
-  ): F[List[ValidatedNel[BadRow, EnrichedEvent]]] = {
-    def flattenToList(
-      v: ValidatedNel[BadRow, Option[ValidatedNel[BadRow, NonEmptyList[EnrichedEvent]]]]
-    ): List[ValidatedNel[BadRow, EnrichedEvent]] =
-      v match {
-        case Validated.Valid(Some(Validated.Valid(nel))) => nel.toList.map(_.valid)
-        case Validated.Valid(Some(Validated.Invalid(f))) => List(f.invalid)
-        case Validated.Invalid(f) => List(f.invalid)
-        case Validated.Valid(None) => Nil
-      }
-
-    val e = for {
-      maybePayload <- input
-    } yield for {
-      payload <- maybePayload
-    } yield (for {
-      events <- EitherT(
+  ): F[List[Validated[BadRow, EnrichedEvent]]] =
+    input match {
+      case Validated.Valid(Some(payload)) =>
         adapterRegistry
           .toRawEvents(payload, client, processor)
-          .map(_.toValidatedNel.toEither)
-      )
-      enrichedEvents <- events.map { e =>
-        EitherT(
-          EnrichmentManager
-            .enrichEvent(enrichmentRegistry, client, processor, etlTstamp, e)
-            .value
-            .map(_.toEitherNel)
-        )
-      }.sequence
-    } yield enrichedEvents).value.map(_.toValidated)
-
-    e.map(_.sequence).sequence.map(flattenToList)
-  }
+          .flatMap {
+            case Validated.Valid(rawEvents) =>
+              rawEvents.toList.traverse { event =>
+                EnrichmentManager
+                  .enrichEvent(
+                    enrichmentRegistry,
+                    client,
+                    processor,
+                    etlTstamp,
+                    event
+                  )
+                  .toValidated
+              }
+            case Validated.Invalid(badRow) =>
+              Monad[F].pure(List(badRow.invalid[EnrichedEvent]))
+          }
+      case Validated.Invalid(badRows) =>
+        Monad[F].pure(badRows.map(_.invalid[EnrichedEvent])).map(_.toList)
+      case Validated.Valid(None) =>
+        Monad[F].pure(List.empty[Validated[BadRow, EnrichedEvent]])
+    }
 }

src/test/scala/com.snowplowanalytics.snowplow.enrich.common/EtlPipelineSpec.scala

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2012-2020 Snowplow Analytics Ltd. All rights reserved.
+ *
+ * This program is licensed to you under the Apache License Version 2.0,
+ * and you may not use this file except in compliance with the Apache License Version 2.0.
+ * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the Apache License Version 2.0 is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+ */
+package com.snowplowanalytics.snowplow.enrich.common
+
+import cats.Id
+import cats.data.Validated
+import cats.syntax.validated._
+
+import com.snowplowanalytics.iglu.client.Client
+import com.snowplowanalytics.iglu.client.resolver.Resolver
+import com.snowplowanalytics.iglu.client.resolver.registries.Registry
+import com.snowplowanalytics.iglu.client.validator.CirceValidator
+import com.snowplowanalytics.snowplow.badrows.{BadRow, Processor}
+
+import io.circe.Json
+
+import org.joda.time.DateTime
+
+import org.specs2.Specification
+import org.specs2.matcher.ValidatedMatchers
+
+import com.snowplowanalytics.snowplow.enrich.common.adapters.AdapterRegistry
+import com.snowplowanalytics.snowplow.enrich.common.enrichments.EnrichmentRegistry
+import com.snowplowanalytics.snowplow.enrich.common.loaders._
+import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent
+import com.snowplowanalytics.snowplow.enrich.common.utils.Clock._
+
+class EtlPipelineSpec extends Specification with ValidatedMatchers {
+  def is = s2"""
+  EtlPipeline should always produce either bad or good row $e1
+  """
+
+  def e1 = {
+    val enrichmentReg = EnrichmentRegistry[Id]()
+    val output = EtlPipelineSpec.commonSetup(enrichmentReg)
+    output must be like {
+      case a :: b :: c :: d :: Nil =>
+        (a must beValid).and(b must beInvalid).and(c must beInvalid).and(d must beInvalid)
+    }
+  }
+}
+
+object EtlPipelineSpec {
+  def commonSetup(enrichmentReg: EnrichmentRegistry[Id]): List[Validated[BadRow, EnrichedEvent]] = {
+    val context =
+      CollectorPayload.Context(
+        Some(DateTime.parse("2017-07-14T03:39:39.000+00:00")),
+        Some("127.0.0.1"),
+        None,
+        None,
+        Nil,
+        None
+      )
+    val source = CollectorPayload.Source("clj-tomcat", "UTF-8", None)
+    val collectorPayload = CollectorPayload(
+      CollectorPayload.Api("com.snowplowanalytics.snowplow", "tp2"),
+      Nil,
+      Some("application/json"),
+      Some(
+        """{
+          |"schema":"iglu:com.snowplowanalytics.snowplow/payload_data/jsonschema/1-0-4",
+          |"data":[
+          |  {"e":"pv","url":"https://console.snowplowanalytics.com/?code=3EaD7VJ2f_aZn33R&state=Y1F6eHNnelUwRVR0c1Q4N3Y3RVM0NlFjdEFLRll6NmsxSnEuZzl%2BTktTQg%3D%3D","page":"Snowplow Insights","refr":"https://id.snowplowanalytics.com/u/login?state=g6Fo2SBDbWRWUmJHTXlHM05pTXhkRTliUUo0YlVIWnVtd0lwUqN0aWTZIHdaQmRyV3NuTTlvNFVZX2tyNUt2MEljcDFGRm9lakFPo2NpZNkgMllzeVFqRHJDTVhoRmRqeXI1MmZ6NDZqUXNlQVpuUUY","tv":"js-2.10.2","tna":"snplow5","aid":"console","p":"web","tz":"America/Chicago","lang":"en-US","cs":"UTF-8","f_pdf":"1","f_qt":"0","f_realp":"0","f_wma":"0","f_dir":"0","f_fla":"0","f_java":"1","f_gears":"0","f_ag":"0","res":"1792x1120","cd":"24","cookie":"1","eid":"1a950884-61d4-4179-a89a-43b67fb58a8f","dtm":"1581382581877","cx":"eyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9jb250ZXh0cy9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6W3sic2NoZW1hIjoiaWdsdTpjb20uZ29vZ2xlLmFuYWx5dGljcy9jb29raWVzL2pzb25zY2hlbWEvMS0wLTAiLCJkYXRhIjp7Il9nYSI6IkdBMS4yLjkwOTgzMDA0OS4xNTgxMDgxMzMzIn19LHsic2NoZW1hIjoiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvd2ViX3BhZ2UvanNvbnNjaGVtYS8xLTAtMCIsImRhdGEiOnsiaWQiOiIxNTNmOTA1ZC1iZGFjLTRhYzUtODI2Yy04YWExNmNmMTNmMzkifX0seyJzY2hlbWEiOiJpZ2x1Om9yZy53My9QZXJmb3JtYW5jZVRpbWluZy9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6eyJuYXZpZ2F0aW9uU3RhcnQiOjE1ODEzODI1ODA3MDYsInVubG9hZEV2ZW50U3RhcnQiOjAsInVubG9hZEV2ZW50RW5kIjowLCJyZWRpcmVjdFN0YXJ0IjowLCJyZWRpcmVjdEVuZCI6MCwiZmV0Y2hTdGFydCI6MTU4MTM4MjU4MTM3OCwiZG9tYWluTG9va3VwU3RhcnQiOjE1ODEzODI1ODEzNzgsImRvbWFpbkxvb2t1cEVuZCI6MTU4MTM4MjU4MTM3OCwiY29ubmVjdFN0YXJ0IjoxNTgxMzgyNTgxMzc4LCJjb25uZWN0RW5kIjoxNTgxMzgyNTgxMzc4LCJzZWN1cmVDb25uZWN0aW9uU3RhcnQiOjAsInJlcXVlc3RTdGFydCI6MTU4MTM4MjU4MTM3OCwicmVzcG9uc2VTdGFydCI6MTU4MTM4MjU4MTUwMCwicmVzcG9uc2VFbmQiOjE1ODEzODI1ODE1MTAsImRvbUxvYWRpbmciOjE1ODEzODI1ODE1MTEsImRvbUludGVyYWN0aXZlIjoxNTgxMzgyNTgxNjg5LCJkb21Db250ZW50TG9hZGVkRXZlbnRTdGFydCI6MTU4MTM4MjU4MTY4OSwiZG9tQ29udGVudExvYWRlZEV2ZW50RW5kIjoxNTgxMzgyNTgxNjg5LCJkb21Db21wbGV0ZSI6MCwibG9hZEV2ZW50U3RhcnQiOjAsImxvYWRFdmVudEVuZCI6MH19XX0","vp":"1792x1034","ds":"1792x1034","vid":"4","sid":"82138304-11e6-4999-92a9-2b99118ec50a","duid":"27b9c270-8741-4b77-8082-14c76341d33e","stm":"1581382583373"},
+          |  {"e":"ue","ue_px":"eyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy91bnN0cnVjdF9ldmVudC9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6eyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9hcHBsaWNhdGlvbl9lcnJvci9qc29uc2NoZW1hLzEtMC0xIiwiZGF0YSI6eyJwcm9ncmFtbWluZ0xhbmd1YWdlIjoiSkFWQVNDUklQVCIsIm1lc3NhZ2UiOiJBUElfRVJST1IgLSA0MDMgLSAiLCJzdGFja1RyYWNlIjpudWxsLCJsaW5lTnVtYmVyIjowLCJsaW5lQ29sdW1uIjowLCJmaWxlTmFtZSI6IiJ9fX0","tv":"js-2.10.2","tna":"snplow5","aid":"console","p":"web","tz":"America/Chicago","lang":"en-US","cs":"UTF-8","f_pdf":"1","f_qt":"0","f_realp":"0","f_wma":"0","f_dir":"0","f_fla":"0","f_java":"1","f_gears":"0","f_ag":"0","res":"1792x1120","cd":"24","cookie":"1","eid":"fed3c6ca-5b8e-47a7-9bfa-102804bd14e0","dtm":"1581382582270","cx":"eyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9jb250ZXh0cy9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6W3sic2NoZW1hIjoiaWdsdTpjb20uZ29vZ2xlLmFuYWx5dGljcy9jb29raWVzL2pzb25zY2hlbWEvMS0wLTAiLCJkYXRhIjp7Il9nYSI6IkdBMS4yLjkwOTgzMDA0OS4xNTgxMDgxMzMzIn19LHsic2NoZW1hIjoiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvd2ViX3BhZ2UvanNvbnNjaGVtYS8xLTAtMCIsImRhdGEiOnsiaWQiOiI0MDVlM2U4My1mOGQ3LTRmM2MtYWE5Zi1jNjQ4MzU5NjRhYzkifX0seyJzY2hlbWEiOiJpZ2x1Om9yZy53My9QZXJmb3JtYW5jZVRpbWluZy9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6eyJuYXZpZ2F0aW9uU3RhcnQiOjE1ODEzODI1ODA3MDYsInVubG9hZEV2ZW50U3RhcnQiOjAsInVubG9hZEV2ZW50RW5kIjowLCJyZWRpcmVjdFN0YXJ0IjowLCJyZWRpcmVjdEVuZCI6MCwiZmV0Y2hTdGFydCI6MTU4MTM4MjU4MTM3OCwiZG9tYWluTG9va3VwU3RhcnQiOjE1ODEzODI1ODEzNzgsImRvbWFpbkxvb2t1cEVuZCI6MTU4MTM4MjU4MTM3OCwiY29ubmVjdFN0YXJ0IjoxNTgxMzgyNTgxMzc4LCJjb25uZWN0RW5kIjoxNTgxMzgyNTgxMzc4LCJzZWN1cmVDb25uZWN0aW9uU3RhcnQiOjAsInJlcXVlc3RTdGFydCI6MTU4MTM4MjU4MTM3OCwicmVzcG9uc2VTdGFydCI6MTU4MTM4MjU4MTUwMCwicmVzcG9uc2VFbmQiOjE1ODEzODI1ODE1MTAsImRvbUxvYWRpbmciOjE1ODEzODI1ODE1MTEsImRvbUludGVyYWN0aXZlIjoxNTgxMzgyNTgxNjg5LCJkb21Db250ZW50TG9hZGVkRXZlbnRTdGFydCI6MTU4MTM4MjU4MTY4OSwiZG9tQ29udGVudExvYWRlZEV2ZW50RW5kIjoxNTgxMzgyNTgxNjg5LCJkb21Db21wbGV0ZSI6MTU4MTM4MjU4MTg3OSwibG9hZEV2ZW50U3RhcnQiOjE1ODEzODI1ODE4NzksImxvYWRFdmVudEVuZCI6MTU4MTM4MjU4MTg3OX19LHsic2NoZW1hIjoiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3MuY29uc29sZS91c2VyL2pzb25zY2hlbWEvMS0wLTAiLCJkYXRhIjp7InVzZXJJZCI6IjIyNTIzOTg1LTUxMzMtNDk1YS05ZTY5LWM3YzIzMDVkOWMxMyIsImZpcnN0TmFtZSI6IlRpIiwibGFzdE5hbWUiOiJMaXB0YWsifX0seyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5jb25zb2xlL29yZ2FuaXphdGlvbi9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6eyJuYW1lIjoiQmlnIFRyZWUiLCJvcmdhbml6YXRpb25JZCI6IjZjZTI0Y2YxLTEzMzAtNGRmYi05YTk0LWVmOTEyNzkxYzE2MCJ9fV19","vp":"1792x1034","ds":"1792x1034","vid":"4","sid":"82138304-11e6-4999-92a9-2b99118ec50a","duid":"27b9c270-8741-4b77-8082-14c76341d33e","refr":"https://console.snowplowanalytics.com/?code=3EaD7VJ2f_aZn33R&state=Y1F6eHNnelUwRVR0c1Q4N3Y3RVM0NlFjdEFLRll6NmsxSnEuZzl%2BTktTQg%3D%3D","url":"https://console.snowplowanalytics.com/","stm":"1581382583373"},
+          |  {"e":"ue","ue_px":"eyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy91bnN0cnVjdF9ldmVudC9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6eyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9hcHBsaWNhdGlvbl9lcnJvci9qc29uc2NoZW1hLzEtMC0xIiwiZGF0YSI6eyJwcm9ncmFtbWluZ0xhbmd1YWdlIjoiSkFWQVNDUklQVCIsIm1lc3NhZ2UiOiJBUElfRVJST1IgLSA0MDMgLSAiLCJzdGFja1RyYWNlIjpudWxsLCJsaW5lTnVtYmVyIjowLCJsaW5lQ29sdW1uIjowLCJmaWxlTmFtZSI6IiJ9fX0","tv":"js-2.10.2","tna":"snplow5","aid":"console","p":"web","tz":"America/Chicago","lang":"en-US","cs":"UTF-8","f_pdf":"1","f_qt":"0","f_realp":"0","f_wma":"0","f_dir":"0","f_fla":"0","f_java":"1","f_gears":"0","f_ag":"0","res":"1792x1120","cd":"24","cookie":"1","eid":"ab8f3859-37cc-4e4a-af9f-3dd91464ee25","dtm":"1581382582276","cx":"eyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9jb250ZXh0cy9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6W3sic2NoZW1hIjoiaWdsdTpjb20uZ29vZ2xlLmFuYWx5dGljcy9jb29raWVzL2pzb25zY2hlbWEvMS0wLTAiLCJkYXRhIjp7Il9nYSI6IkdBMS4yLjkwOTgzMDA0OS4xNTgxMDgxMzMzIn19LHsic2NoZW1hIjoiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvd2ViX3BhZ2UvanNvbnNjaGVtYS8xLTAtMCIsImRhdGEiOnsiaWQiOiI0MDVlM2U4My1mOGQ3LTRmM2MtYWE5Zi1jNjQ4MzU5NjRhYzkifX0seyJzY2hlbWEiOiJpZ2x1Om9yZy53My9QZXJmb3JtYW5jZVRpbWluZy9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6eyJuYXZpZ2F0aW9uU3RhcnQiOjE1ODEzODI1ODA3MDYsInVubG9hZEV2ZW50U3RhcnQiOjAsInVubG9hZEV2ZW50RW5kIjowLCJyZWRpcmVjdFN0YXJ0IjowLCJyZWRpcmVjdEVuZCI6MCwiZmV0Y2hTdGFydCI6MTU4MTM4MjU4MTM3OCwiZG9tYWluTG9va3VwU3RhcnQiOjE1ODEzODI1ODEzNzgsImRvbWFpbkxvb2t1cEVuZCI6MTU4MTM4MjU4MTM3OCwiY29ubmVjdFN0YXJ0IjoxNTgxMzgyNTgxMzc4LCJjb25uZWN0RW5kIjoxNTgxMzgyNTgxMzc4LCJzZWN1cmVDb25uZWN0aW9uU3RhcnQiOjAsInJlcXVlc3RTdGFydCI6MTU4MTM4MjU4MTM3OCwicmVzcG9uc2VTdGFydCI6MTU4MTM4MjU4MTUwMCwicmVzcG9uc2VFbmQiOjE1ODEzODI1ODE1MTAsImRvbUxvYWRpbmciOjE1ODEzODI1ODE1MTEsImRvbUludGVyYWN0aXZlIjoxNTgxMzgyNTgxNjg5LCJkb21Db250ZW50TG9hZGVkRXZlbnRTdGFydCI6MTU4MTM4MjU4MTY4OSwiZG9tQ29udGVudExvYWRlZEV2ZW50RW5kIjoxNTgxMzgyNTgxNjg5LCJkb21Db21wbGV0ZSI6MTU4MTM4MjU4MTg3OSwibG9hZEV2ZW50U3RhcnQiOjE1ODEzODI1ODE4NzksImxvYWRFdmVudEVuZCI6MTU4MTM4MjU4MTg3OX19LHsic2NoZW1hIjoiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3MuY29uc29sZS91c2VyL2pzb25zY2hlbWEvMS0wLTAiLCJkYXRhIjp7InVzZXJJZCI6IjIyNTIzOTg1LTUxMzMtNDk1YS05ZTY5LWM3YzIzMDVkOWMxMyIsImZpcnN0TmFtZSI6IlRpIiwibGFzdE5hbWUiOiJMaXB0YWsifX0seyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5jb25zb2xlL29yZ2FuaXphdGlvbi9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6eyJuYW1lIjoiQmlnIFRyZWUiLCJvcmdhbml6YXRpb25JZCI6IjZjZTI0Y2YxLTEzMzAtNGRmYi05YTk0LWVmOTEyNzkxYzE2MCJ9fV19","vp":"1792x1034","ds":"1792x1034","vid":"4","sid":"82138304-11e6-4999-92a9-2b99118ec50a","duid":"27b9c270-8741-4b77-8082-14c76341d33e","refr":"https://console.snowplowanalytics.com/?code=3EaD7VJ2f_aZn33R&state=Y1F6eHNnelUwRVR0c1Q4N3Y3RVM0NlFjdEFLRll6NmsxSnEuZzl%2BTktTQg%3D%3D","url":"https://console.snowplowanalytics.com/","stm":"1581382583373"},
+          |  {"e":"ue","ue_px":"eyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy91bnN0cnVjdF9ldmVudC9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6eyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9hcHBsaWNhdGlvbl9lcnJvci9qc29uc2NoZW1hLzEtMC0xIiwiZGF0YSI6eyJwcm9ncmFtbWluZ0xhbmd1YWdlIjoiSkFWQVNDUklQVCIsIm1lc3NhZ2UiOiJBUElfRVJST1IgLSA0MDMgLSAiLCJzdGFja1RyYWNlIjpudWxsLCJsaW5lTnVtYmVyIjowLCJsaW5lQ29sdW1uIjowLCJmaWxlTmFtZSI6IiJ9fX0","tv":"js-2.10.2","tna":"snplow5","aid":"console","p":"web","tz":"America/Chicago","lang":"en-US","cs":"UTF-8","f_pdf":"1","f_qt":"0","f_realp":"0","f_wma":"0","f_dir":"0","f_fla":"0","f_java":"1","f_gears":"0","f_ag":"0","res":"1792x1120","cd":"24","cookie":"1","eid":"e94b4262-63e5-4811-b5db-21c4bd2ab354","dtm":"1581382582388","cx":"eyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9jb250ZXh0cy9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6W3sic2NoZW1hIjoiaWdsdTpjb20uZ29vZ2xlLmFuYWx5dGljcy9jb29raWVzL2pzb25zY2hlbWEvMS0wLTAiLCJkYXRhIjp7Il9nYSI6IkdBMS4yLjkwOTgzMDA0OS4xNTgxMDgxMzMzIn19LHsic2NoZW1hIjoiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvd2ViX3BhZ2UvanNvbnNjaGVtYS8xLTAtMCIsImRhdGEiOnsiaWQiOiI0MDVlM2U4My1mOGQ3LTRmM2MtYWE5Zi1jNjQ4MzU5NjRhYzkifX0seyJzY2hlbWEiOiJpZ2x1Om9yZy53My9QZXJmb3JtYW5jZVRpbWluZy9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6eyJuYXZpZ2F0aW9uU3RhcnQiOjE1ODEzODI1ODA3MDYsInVubG9hZEV2ZW50U3RhcnQiOjAsInVubG9hZEV2ZW50RW5kIjowLCJyZWRpcmVjdFN0YXJ0IjowLCJyZWRpcmVjdEVuZCI6MCwiZmV0Y2hTdGFydCI6MTU4MTM4MjU4MTM3OCwiZG9tYWluTG9va3VwU3RhcnQiOjE1ODEzODI1ODEzNzgsImRvbWFpbkxvb2t1cEVuZCI6MTU4MTM4MjU4MTM3OCwiY29ubmVjdFN0YXJ0IjoxNTgxMzgyNTgxMzc4LCJjb25uZWN0RW5kIjoxNTgxMzgyNTgxMzc4LCJzZWN1cmVDb25uZWN0aW9uU3RhcnQiOjAsInJlcXVlc3RTdGFydCI6MTU4MTM4MjU4MTM3OCwicmVzcG9uc2VTdGFydCI6MTU4MTM4MjU4MTUwMCwicmVzcG9uc2VFbmQiOjE1ODEzODI1ODE1MTAsImRvbUxvYWRpbmciOjE1ODEzODI1ODE1MTEsImRvbUludGVyYWN0aXZlIjoxNTgxMzgyNTgxNjg5LCJkb21Db250ZW50TG9hZGVkRXZlbnRTdGFydCI6MTU4MTM4MjU4MTY4OSwiZG9tQ29udGVudExvYWRlZEV2ZW50RW5kIjoxNTgxMzgyNTgxNjg5LCJkb21Db21wbGV0ZSI6MTU4MTM4MjU4MTg3OSwibG9hZEV2ZW50U3RhcnQiOjE1ODEzODI1ODE4NzksImxvYWRFdmVudEVuZCI6MTU4MTM4MjU4MTg3OX19LHsic2NoZW1hIjoiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3MuY29uc29sZS91c2VyL2pzb25zY2hlbWEvMS0wLTAiLCJkYXRhIjp7InVzZXJJZCI6IjIyNTIzOTg1LTUxMzMtNDk1YS05ZTY5LWM3YzIzMDVkOWMxMyIsImZpcnN0TmFtZSI6IlRpIiwibGFzdE5hbWUiOiJMaXB0YWsifX0seyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5jb25zb2xlL29yZ2FuaXphdGlvbi9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6eyJuYW1lIjoiQmlnIFRyZWUiLCJvcmdhbml6YXRpb25JZCI6IjZjZTI0Y2YxLTEzMzAtNGRmYi05YTk0LWVmOTEyNzkxYzE2MCJ9fV19","vp":"1792x1034","ds":"1792x1034","vid":"4","sid":"82138304-11e6-4999-92a9-2b99118ec50a","duid":"27b9c270-8741-4b77-8082-14c76341d33e","refr":"https://console.snowplowanalytics.com/?code=3EaD7VJ2f_aZn33R&state=Y1F6eHNnelUwRVR0c1Q4N3Y3RVM0NlFjdEFLRll6NmsxSnEuZzl%2BTktTQg%3D%3D","url":"https://console.snowplowanalytics.com/","stm":"1581382583373"}
+          |]}""".stripMargin
+      ),
+      source,
+      context
+    )
+    val input = Some(collectorPayload).validNel
+    val reg = Registry.IgluCentral
+    val client = Client[Id, Json](Resolver(List(reg), None), CirceValidator)
+    EtlPipeline
+      .processEvents[Id](
+        new AdapterRegistry(),
+        enrichmentReg,
+        client,
+        Processor("sce-test-suite", "1.0.0"),
+        new DateTime(1500000000L),
+        input
+      )
+  }
+}

src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala

@@ -16,7 +16,7 @@ package registry
 package pii
 
 import cats.Id
-import cats.data.ValidatedNel
+import cats.data.Validated
 import cats.syntax.option._
 import cats.syntax.validated._
 
@@ -57,9 +57,7 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidatedMatcher
   Hashing configured JSON fields in POJO should not create new fields                                         $e8
   """
 
-  def commonSetup(
-    enrichmentReg: EnrichmentRegistry[Id]
-  ): List[ValidatedNel[BadRow, EnrichedEvent]] = {
+  def commonSetup(enrichmentReg: EnrichmentRegistry[Id]): List[Validated[BadRow, EnrichedEvent]] = {
     val context =
       CollectorPayload.Context(
         Some(DateTime.parse("2017-07-14T03:39:39.000+00:00")),