From c678acccc7c25a075770fa4a35fdb7e056812325 Mon Sep 17 00:00:00 2001 From: dafreels Date: Mon, 30 Aug 2021 07:04:20 -0400 Subject: [PATCH 01/24] #253 Fixed an issue where the default credential parser for the secrets manager credential providers was not being loaded. --- metalus-application/pom.xml | 2 +- metalus-aws/pom.xml | 2 +- .../pipeline/AWSSecretsManagerCredentialProvider.scala | 8 ++++---- metalus-common/pom.xml | 2 +- metalus-core/pom.xml | 2 +- .../scala/com/acxiom/pipeline/CredentialProvider.scala | 5 +++-- metalus-delta/pom.xml | 2 +- metalus-examples/pom.xml | 2 +- metalus-gcp/pom.xml | 2 +- .../pipeline/GCPSecretsManagerCredentialProvider.scala | 6 +++--- metalus-kafka/pom.xml | 2 +- metalus-mongo/pom.xml | 2 +- metalus-utils/pom.xml | 2 +- pom.xml | 2 +- 14 files changed, 21 insertions(+), 20 deletions(-) diff --git a/metalus-application/pom.xml b/metalus-application/pom.xml index fed018a3..645ed1b4 100644 --- a/metalus-application/pom.xml +++ b/metalus-application/pom.xml @@ -9,7 +9,7 @@ com.acxiom metalus - 1.8.2-SNAPSHOT + 1.8.3-SNAPSHOT diff --git a/metalus-aws/pom.xml b/metalus-aws/pom.xml index f8270771..4d72791c 100644 --- a/metalus-aws/pom.xml +++ b/metalus-aws/pom.xml @@ -9,7 +9,7 @@ com.acxiom metalus - 1.8.2-SNAPSHOT + 1.8.3-SNAPSHOT diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/AWSSecretsManagerCredentialProvider.scala b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/AWSSecretsManagerCredentialProvider.scala index 65f467d7..9847e0ae 100644 --- a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/AWSSecretsManagerCredentialProvider.scala +++ b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/AWSSecretsManagerCredentialProvider.scala @@ -1,16 +1,16 @@ package com.acxiom.aws.pipeline import com.acxiom.aws.utils.{AWSBasicCredential, AWSCloudWatchCredential, AWSDynamoDBCredential, DefaultAWSCredential} -import com.acxiom.pipeline.{Credential, CredentialParser, DefaultCredentialProvider} +import com.acxiom.pipeline.{Credential, CredentialParser, DefaultCredentialParser, DefaultCredentialProvider} import com.amazonaws.client.builder.AwsClientBuilder import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder import com.amazonaws.services.secretsmanager.model.{GetSecretValueRequest, InvalidParameterException, InvalidRequestException, ResourceNotFoundException} import org.apache.log4j.Logger -class AWSSecretsManagerCredentialProvider(val params: Map[String, Any]) - extends DefaultCredentialProvider(params + - ("credential-parsers" -> s"${params.getOrElse("credential-parsers", "")},com.acxiom.aws.pipeline.AWSCredentialParser")) { +class AWSSecretsManagerCredentialProvider(override val parameters: Map[String, Any]) + extends DefaultCredentialProvider(parameters) { private val logger = Logger.getLogger(getClass) + override protected val defaultParsers = List(new DefaultCredentialParser(), new AWSCredentialParser) val region: String = parameters.getOrElse("region", "us-east-1").asInstanceOf[String] private val config = new AwsClientBuilder.EndpointConfiguration(s"secretsmanager.$region.amazonaws.com", region) private val clientBuilder = AWSSecretsManagerClientBuilder.standard diff --git a/metalus-common/pom.xml b/metalus-common/pom.xml index 3c102eaa..eba9aab5 100644 --- a/metalus-common/pom.xml +++ b/metalus-common/pom.xml @@ -9,7 +9,7 @@ com.acxiom metalus - 1.8.2-SNAPSHOT + 1.8.3-SNAPSHOT diff --git a/metalus-core/pom.xml b/metalus-core/pom.xml index 000cdda4..dc78fdf5 100644 --- a/metalus-core/pom.xml +++ b/metalus-core/pom.xml @@ -9,7 +9,7 @@ com.acxiom metalus - 1.8.2-SNAPSHOT + 1.8.3-SNAPSHOT diff --git a/metalus-core/src/main/scala/com/acxiom/pipeline/CredentialProvider.scala b/metalus-core/src/main/scala/com/acxiom/pipeline/CredentialProvider.scala index af896f6d..b1a0727a 100644 --- a/metalus-core/src/main/scala/com/acxiom/pipeline/CredentialProvider.scala +++ b/metalus-core/src/main/scala/com/acxiom/pipeline/CredentialProvider.scala @@ -60,14 +60,15 @@ class DefaultCredentialParser extends CredentialParser { * @param parameters A map containing parameters. */ class DefaultCredentialProvider(override val parameters: Map[String, Any]) extends CredentialProvider { + protected val defaultParsers: List[CredentialParser] = List(new DefaultCredentialParser()) protected lazy val credentialParsers: List[CredentialParser] = { if (parameters.contains("credential-parsers")) { parameters("credential-parsers").asInstanceOf[String] .split(',').filter(_.nonEmpty).distinct.map(className => ReflectionUtils.loadClass(className, Some(Map("parameters" -> parameters))) - .asInstanceOf[CredentialParser]).toList:+ new DefaultCredentialParser() + .asInstanceOf[CredentialParser]).toList ++ defaultParsers } else { - List(new DefaultCredentialParser()) + defaultParsers } } protected lazy val credentials: Map[String, Credential] = { diff --git a/metalus-delta/pom.xml b/metalus-delta/pom.xml index 7cfeed96..3a3bbc9e 100644 --- a/metalus-delta/pom.xml +++ b/metalus-delta/pom.xml @@ -12,7 +12,7 @@ com.acxiom metalus - 1.8.2-SNAPSHOT + 1.8.3-SNAPSHOT diff --git a/metalus-examples/pom.xml b/metalus-examples/pom.xml index 53d04109..90d526f0 100644 --- a/metalus-examples/pom.xml +++ b/metalus-examples/pom.xml @@ -9,7 +9,7 @@ com.acxiom metalus - 1.8.2-SNAPSHOT + 1.8.3-SNAPSHOT diff --git a/metalus-gcp/pom.xml b/metalus-gcp/pom.xml index e4cebf26..f580bb98 100644 --- a/metalus-gcp/pom.xml +++ b/metalus-gcp/pom.xml @@ -9,7 +9,7 @@ com.acxiom metalus - 1.8.2-SNAPSHOT + 1.8.3-SNAPSHOT diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPSecretsManagerCredentialProvider.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPSecretsManagerCredentialProvider.scala index 6d49b8fa..5c39234b 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPSecretsManagerCredentialProvider.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPSecretsManagerCredentialProvider.scala @@ -1,6 +1,6 @@ package com.acxiom.gcp.pipeline -import com.acxiom.pipeline.{Credential, CredentialParser, DefaultCredential, DefaultCredentialProvider} +import com.acxiom.pipeline._ import com.google.cloud.secretmanager.v1.{SecretManagerServiceClient, SecretName, SecretVersion, SecretVersionName} import org.apache.log4j.Logger @@ -13,9 +13,9 @@ import scala.collection.JavaConverters._ * @param parameters A map containing parameters. projectId is required. */ class GCPSecretsManagerCredentialProvider(override val parameters: Map[String, Any]) - extends DefaultCredentialProvider(parameters + - ("credential-parsers" -> s"${parameters.getOrElse("credential-parsers", "")},com.acxiom.gcp.pipeline.GCPCredentialParser")) { + extends DefaultCredentialProvider(parameters) { private val logger = Logger.getLogger(getClass) + override protected val defaultParsers = List(new DefaultCredentialParser(), new GCPCredentialParser) private val projectId = parameters.getOrElse("projectId", "").asInstanceOf[String] private val secretsManagerClient = SecretManagerServiceClient.create() diff --git a/metalus-kafka/pom.xml b/metalus-kafka/pom.xml index 95005824..74515660 100644 --- a/metalus-kafka/pom.xml +++ b/metalus-kafka/pom.xml @@ -9,7 +9,7 @@ com.acxiom metalus - 1.8.2-SNAPSHOT + 1.8.3-SNAPSHOT diff --git a/metalus-mongo/pom.xml b/metalus-mongo/pom.xml index 7f43acf6..4252d0e0 100644 --- a/metalus-mongo/pom.xml +++ b/metalus-mongo/pom.xml @@ -9,7 +9,7 @@ com.acxiom metalus - 1.8.2-SNAPSHOT + 1.8.3-SNAPSHOT diff --git a/metalus-utils/pom.xml b/metalus-utils/pom.xml index 3a13dacc..28553835 100644 --- a/metalus-utils/pom.xml +++ b/metalus-utils/pom.xml @@ -9,7 +9,7 @@ com.acxiom metalus - 1.8.2-SNAPSHOT + 1.8.3-SNAPSHOT diff --git a/pom.xml b/pom.xml index f5f315ba..4cf6fda8 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.acxiom metalus - 1.8.2-SNAPSHOT + 1.8.3-SNAPSHOT ${project.artifactId} pom Metalus Pipeline Library From bb37150f28ebdd9869d6edf9c025132011aa9357 Mon Sep 17 00:00:00 2001 From: dafreels Date: Wed, 1 Sep 2021 11:42:40 -0400 Subject: [PATCH 02/24] Updated build file to fix issue with release step failing and added debug log to determine why onr jar isn't uploading. --- .github/workflows/build.yml | 36 +++++------------------------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 971e7202..ea2d5343 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,6 +142,10 @@ jobs: echo 'METALUS_VERSION<> $GITHUB_ENV mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec >> $GITHUB_ENV echo 'EOF' >> $GITHUB_ENV + - name: Display Metalus Application Target Directory + run: | + pwd + ls -l metalus-application/target - name: Upload Application Jar uses: actions/upload-artifact@v2 with: @@ -177,38 +181,8 @@ jobs: echo 'METALUS_VERSION<> $GITHUB_ENV mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec >> $GITHUB_ENV echo 'EOF' >> $GITHUB_ENV - - name: Download Application Jar Spark 2.4 - uses: actions/download-artifact@v2 - with: - name: metalus-application_2.11-spark_2.4-${{ env.METALUS_VERSION }}.jar - - name: Download Utils Spark 2.4 - uses: actions/download-artifact@v2 - with: - name: metalus-utils_2.11-spark_2.4-${{ env.METALUS_VERSION }}.tar.gz - - name: Download Application Jar Spark 2.4 Scala 2.12 - uses: actions/download-artifact@v2 - with: - name: metalus-application_2.12-spark_2.4-${{ env.METALUS_VERSION }}.jar - - name: Download Utils Spark 2.4 Scala 2.12 - uses: actions/download-artifact@v2 - with: - name: metalus-utils_2.12-spark_2.4-${{ env.METALUS_VERSION }}.tar.gz - - name: Download Application Jar Spark 3.0 - uses: actions/download-artifact@v2 - with: - name: metalus-application_2.12-spark_3.0-${{ env.METALUS_VERSION }}.jar - - name: Download Utils Spark 3.0 - uses: actions/download-artifact@v2 - with: - name: metalus-utils_2.12-spark_3.0-${{ env.METALUS_VERSION }}.tar.gz - - name: Download Application Jar Spark 3.1 - uses: actions/download-artifact@v2 - with: - name: metalus-application_2.12-spark_3.1-${{ env.METALUS_VERSION }}.jar - - name: Download Utils Spark 3.1 + - name: Download Artifacts uses: actions/download-artifact@v2 - with: - name: metalus-utils_2.12-spark_3.1-${{ env.METALUS_VERSION }}.tar.gz - name: Create Github Release uses: "marvinpinto/action-automatic-releases@latest" with: From c603dc38c84d3a846317d8b1447a2a21fb4a17e8 Mon Sep 17 00:00:00 2001 From: dafreels Date: Tue, 7 Sep 2021 09:39:07 -0400 Subject: [PATCH 03/24] #252 Implemented DataConnectorSteps to allow generic read/write using a new data connectors framework. HDFS, S3 and GCS batch connectors have been implemented. --- docs/dataconnectors.md | 105 +++++++++ docs/images/DataConnectors.png | Bin 0 -> 47323 bytes docs/readme.md | 2 + .../awssecretsmanager-credentialprovider.md | 22 +- .../AWSSecretsManagerCredentialProvider.scala | 17 +- .../pipeline/connectors/S3DataConnector.scala | 43 ++++ .../scala/com/acxiom/aws/steps/S3Steps.scala | 9 +- .../com/acxiom/aws/utils/AWSCredential.scala | 44 +++- metalus-common/docs/dataconnectorsteps.md | 19 ++ metalus-common/readme.md | 1 + .../pipeline/connectors/DataConnector.scala | 20 ++ .../connectors/DataConnectorUtilities.scala | 54 +++++ .../connectors/HDFSDataConnector.scala | 18 ++ .../pipeline/steps/DataConnectorSteps.scala | 37 +++ .../pipeline/steps/DataFrameSteps.scala | 53 +---- .../steps/DataConnectorStepsTests.scala | 214 ++++++++++++++++++ .../acxiom/pipeline/CredentialProvider.scala | 12 +- .../gcpsecretsmanager-credentialprovider.md | 6 + .../acxiom/gcp/pipeline/GCPCredential.scala | 8 + .../GCPSecretsManagerCredentialProvider.scala | 15 +- .../connectors/GCSDataConnector.scala | 40 ++++ .../scala/com/acxiom/gcp/steps/GCSSteps.scala | 31 +-- .../com/acxiom/gcp/utils/GCPUtilities.scala | 18 ++ 23 files changed, 696 insertions(+), 92 deletions(-) create mode 100644 docs/dataconnectors.md create mode 100644 docs/images/DataConnectors.png create mode 100644 metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3DataConnector.scala create mode 100644 metalus-common/docs/dataconnectorsteps.md create mode 100644 metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala create mode 100644 metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnectorUtilities.scala create mode 100644 metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSDataConnector.scala create mode 100644 metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataConnectorSteps.scala create mode 100644 metalus-common/src/test/scala/com/acxiom/pipeline/steps/DataConnectorStepsTests.scala create mode 100644 metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala diff --git a/docs/dataconnectors.md b/docs/dataconnectors.md new file mode 100644 index 00000000..9aa0e298 --- /dev/null +++ b/docs/dataconnectors.md @@ -0,0 +1,105 @@ +[Documentation Home](readme.md) + +# Data Connectors +Data Connectors provide an abstraction for loading and writing data. This is useful for creating generic pipelines that +can used across providers without source/destination knowledge prior to runtime. Each connector has the responsibility +to load and write a DataFrame based on the underlying system. Below is a breakdown of how connectors may be classified: + +![DataConnectors](images/DataConnectors.png) + +## Batch +Connectors that are designed to load and write data for batch processing will extend the _BatchDataConnector_. These +are very straightforward and offer the most reusable components. + +### HDFSDataConnector +This connector provides access to HDFS. The _credentialName_ and _credential_ parameters are not used in this implementation, +instead relying on the permissions of the cluster. Below is an example setup: + +#### Scala +```scala +val connector = HDFSDataConnector("my-connector", None, None, + DataFrameReaderOptions(format = "csv"), + DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) +``` +#### Globals JSON +```json +{ + "connector": { + "className": "com.acxiom.pipeline.connectors.HDFSDataConnector", + "object": { + "name": "my-connector", + "readOptions": { + "format": "csv" + }, + "writeOptions": { + "format": "csv", + "options": { + "delimiter": "þ" + } + } + } + } +} +``` +### S3DataConnector +This connector provides access to S3. Below is an example setup that expects a secrets manager credential provider: +#### Scala +```scala +val connector = S3DataConnector("my-connector", Some("my-credential-name-for-secrets-manager"), None, + DataFrameReaderOptions(format = "csv"), + DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) +``` +#### Globals JSON +```json +{ + "connector": { + "className": "com.acxiom.aws.pipeline.connectors.S3DataConnector", + "object": { + "name": "my-connector", + "credentialName": "my-credential-name-for-secrets-manager", + "readOptions": { + "format": "csv" + }, + "writeOptions": { + "format": "csv", + "options": { + "delimiter": "þ" + } + } + } + } +} +``` +### GCSDataConnector +This connector provides access to GCS. Below is an example setup that expects a secrets manager credential provider: +#### Scala +```scala +val connector = GCSDataConnector("my-connector", Some("my-credential-name-for-secrets-manager"), None, + DataFrameReaderOptions(format = "csv"), + DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) +``` +#### Globals JSON +```json +{ + "connector": { + "className": "com.acxiom.gcp.pipeline.connectors.GCSDataConnector", + "object": { + "name": "my-connector", + "credentialName": "my-credential-name-for-secrets-manager", + "readOptions": { + "format": "csv" + }, + "writeOptions": { + "format": "csv", + "options": { + "delimiter": "þ" + } + } + } + } +} +``` +## Streaming (Coming Soon) +Streaming connectors offer a way to use pipelines with [Spark Structured Streaming](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html) without +the need to write new [drivers](pipeline-drivers.md). When designing pipelines for streaming, care must be taken to not +inject steps that are more batch oriented such as doing a file copy. diff --git a/docs/images/DataConnectors.png b/docs/images/DataConnectors.png new file mode 100644 index 0000000000000000000000000000000000000000..57ac81bf609e1fe09624d2dbdd5c2277dbd36a6b GIT binary patch literal 47323 zcmeFYXH-*L+b9})FDQy2!nOgSN-t6@6eSc1y@N>aT}puM1}jA+1gTLlK?QFqDKMB?JiNuCT@ZzT-XP-tXM;-FyCg`J-#DXFk2oHF@zsOYOu_j-wzD=*0bd z%1{vK2m}P$%Q$=xSZOm*6$gRBK=+kz>pdQr8CH63W*j0ldXy)zHCq1D-KOZCljkyTUz`mKcTdTpI}0zmtJe*dF^|6>}sF|xEBMAK6K zAvLCL^@|Iu0#+$yWn*TfFnfbOF!~YEf;>^Edz(cR32_E>hRpPd>x-uN~>nV(f5Ax#qZ@lgMZNV%$ zh`qnwCcYQEegzLi^3Lzp+5aKRRD|Y({}bw!`BB_W2wiCQ58W&em41f|Su61u+4_zg z*&O$;gVB>n+aH=1n+ozn{Yy?7$0$`VO@fg_8}7!)+^DzF|9F?i$!fv4&jiLZ8M)Af z_)uTn!U20ve!BiqQ-)cRVcpi;#9Fj*JUp{l7%x{ITO+;5wKf^Lt!)eb?UAEWmvoIO zy!5(N8GWbXkDPEXydsy)94|cqR~o}$%!Y2xw|>pSP7a$R#cZ>;Xu)-RyFzr;T^WJe z{J)+2xD2AW`$5MwT)V9klmth?U;1rcfOtA`+oD-hppPkvxS{#>uW1_$Y=NQGfaPMM zP|zuX-@9j;isDgKYw_jVo9QpqEfrx!@#xzOzRToFALQUW$!P(jS;n_|M64J|*;xH| zoXH%|)I)IaYn;%uP!CyRC=F?3;9+r49oaTvdcVJmIhME1AbJ!6P2}Hp>tM~*NTYDY z8b-I88d%}I-{l&?IZR?z&XzCA-~)dIE!He2;{c(#)}A(qAhbwtBQ ziMg`+PBULWR=w8ak&Xka#b$Tw7uc64#|}~y%Y$K-ECSmhI>y-+<6;ZWow|Q;EOGJa zCQ=4d1i{%T2ckK0alL0g)@_>qK=s`IqV=un12seLr1khFeb_wG))tNRh5ieR*?cb0dT6pF< zZ7S&zTyjn^9y@8mEz~Rh2Cbn^LA;`*8Jmybdo>UmF?btOsXqlRyUdf^9Nk)2p4p6i zp>-FoD^hSBqN4XF@M5w)=0!T^!iIJ@cUd{L^iMB2?gR@uaLl% zyuke=eq9vhOj~fPCc5R1#8#w;V2Ce7_EpA)AphrX-3Ux&BTYHY zmS8RRUUgb zDJ0foo1$_(Q-y9Ek2-&@<>?4Fts8Ain4YX#wVj{!;cjjZ&(S1(8D?Bq;+n^1vc>ga zV0!7l%aO+!6ZxkF!eBDS+j97dx}{W#U>u|AD{^j$(p-F%=%+y zlSCJ^VBFpZ^lufmX!o2D;eeSzlkvGXk8Ow*rapM(j3V8NMg+FKdb@g)G?~)zs z3wZb(ok)J;X3Ts2+nDI@H$N74 zW1>Hr`nsMZUr_v=$;0+!8X@pL@9$>{>Ta>)^TDE(qL9Jix zJ}{&w1|{qprD^VGS}H{@YqFu_Hy6`?_R;TI9?4}q{vBf=RhhVAy_Z76XWAZplE|DZ z$jlYXcqcXuKID6hAAGgPHWBW_a}_T|yO#T*Q^VW^AD*kym972>ZCXq*jh!nvm4|!x zknkv~SP%Abw8|C@|MB#t+bcI-YOHU|QWL$~d8tbz6yxJcd168LweSmmXbv~k8!?rD zY5cA$R^c|n5Y!)U_d@7KwQf(38VnD%(uvnI;FeIU`&H%UBnyRrtSwrVhdy@+f+1v^ zye@k@?tN7ZtW(xN#JK~;RXM-GaAUFW)a9oLFmf4^v66rzB<7hSvLUK8=h!sprrad5 z8kLQu-!XrXgY-Y@Ib3XPK~Pbt!1k6^a|P5c4&((M|0itf596Q+lk4CD!Nhi5?cl^< zzRGLEUOGqleT{iy(5COunNphBVvtQv{Qh(|*=)E^RYQt=bntw!S1bRtfJ|JsZQi0x z2@kD+jAXdk6ssC%3%>@qq1<#Tu*fXVs3fe*(lifVl#4DJ(lS?WR_ZMz^P3(uc|K>VB#Dz(dIG3#aTKU-$_GcSoLo(d zSJ=TRfjcyX@;P{sN{sgu->rYVo8@GsVB8g``i)o645d78#u^rG1r-LLeYWlNJyiw%t*B)F~&Ygw{qF*wy1A2$Yvk~Sw9 zRfNnTV(}dvy00;2Zn(F=g8ISIba~5Ynd=@4I@r}eYEe$GItE(?F0e695-r!M+qz5A zC1wP12f&s#Us*Kk6E%PPjRWy=JWBaT=|8>s5dZ(Q-#~-?KZC~3y!HRrpmE3C;xtRY z;l`2Uf0G5x#Qon6q(?9Q%Qy+z2e`5brQ`QEb`Ty?>|gqOc%Py?S$uZqahfOD*u!Lw7_?EG^%epAv*)hxi~ zWvVIER#cdV)Vfo;m!>!5wnr%cY`?5F+oOWkc$-}(tTx|cnR(+M$`J+F%9=X$J=q0D zsDD{5)qe$5A-#dEVKYnmFXikW5r8=1yduKf-4U7gC2Mq>W7@C?bkclXMTN8PF14_~ zZW#HPc#7%GhokBDS(^EOtGs;T`BW8T`o?d_)uMnW8;ez!sk|;l5;c z%9{WDE|x;xQ`~KeO2}5id2So&&?HA)+)52XGZF9Irt@UYWPblHpp1?UUKwZf)?_jO zadkaD|uoaRBheN{E^3+y9~X5dtW)m z5?dADq@3`e6wl&;mtoL8e8W8S#CyVafh2M{mk>5fjR-fqjyrh9=+j{N;{HNPg`$e| zE{K)^&oSH2?Q-Nqjnv{N&jcDWS>GFg>*CP5_}}W}tcAWeCO_U0UK)o<_`|mpU-UAE zvo}+I*E01TQ;2i}p{7Sy-aKh>G?}cxVi&vFO8V^$&iLqvefO%52 z!S0D**lljSj=d!zeF@xsl2{=(ApL=d+> zrVdxg`!0>t2FB81Zp}rncU8j6gB$hDR8>f7t+On%dOP35VlRC*kX1tl1lA^R0-&Jh%B8SYbZd&fo8s%;Y(IAylFUYcIO0aF7DYVlP9I8b(kZ=$ zn}P8?aNS_=9cW-Zr)jmD=K5}BwihiZCu<+Q#4}bO3%x&kSHNqXs-bH8|O|L7WbL+(IiiN*&gHf$ITiAp-3 zMLDAnma)rD?%6F*(OS_f{A$yv3uSrZwO(fL?i?7x=5fXMFEJEC-S~8NwS5R>`V#*& zK@*}Q9T0jEtPU_RSdRDLggWt`%?f$P{k|{MMR)zt z?RLm5BzJc_R8L)FFoo5Z)~I)GR8Q}EcF6{N{8;R7O>@2mru0FjArVv-XEV@q!^jni zIVV@W4PNZlKEH>(I8`!6aiefcXYMLJj#0V@`r>Dtwm2#FiyK}M-K52R3mc;fUC9{p zmlUlV3t#-hhO;1ggYpOUcM*l9FZoDS>&B;H0(>k&Zp<6;hJ9g_Y%eq$(iiQQ)_k@C z*{)}Mxi|qQnf5)uY}c1$zGYPdPxGn{xp8uI{w0g<0VS<;{I$CRr~am0Z{ip5qw|!~ z)K*vN>!Elk>#cpE@`Qxx(`nAt3Kofx+eBl)Dzx>ry=*^^DIr$CqudaoAGB{||RvZMEA1I5%P zpEYsENEK5+Lr*m#_(LaW(+t6Hvd*Y7!RB~dSW$s_)-cz6Ocaxhu>H4>5=;@h&_D# zU_KVIUc6_s6uBzJlZJ~HK^|3AxOH-$!f*URWz_T~XrWi|Nl@WE(hv3DFPtm$(w8_R z_JLfo-3Q2+?VF9ediMzDOgRK9{kr?y4pN)k13K=#OP7?H6ELU`_y zQETW30D)Xg6`~}zQ>J%~x$>F0%K1|4lZ4vrZxUs)a;HFHHpG-blW5xDN+@C1Jx0ID zFyA0{sf>PQLV^&x!ql^P`f!0jGmU^ShQfCW*LQfk)y32R=$SPMw~opp|DrwmgNX|i zJ&28_QMdeIvP43|$sPFm+g?m8rMO=ztH1Bfd@hXqWy!Gg?@ZaO3^}O)THrU7xKTLz zT0zMR*=fnUay&3)WKJ2RDOC6<+KdhH* zgm$>(Or#jqiT|?et$p}YhrA9f1O3kcQ&G2;kg+QPxg?h@KDM}DRj(dl`(^vf!r2%p zg}}6GbO9@1K8dADr)Riz-)Md%@i%NcJQ}>tc`H7Q=HxBv&~NWr^Uco?R#P=H2RPym zJvi7@6f#V(5!Ty@>aeD3h8N?nUh%SB;%UfonTStcHfsV#sKQEDZmJ>A=EPoh9DN7U zV-eqjS8^#X2t|3(pFeXc7ElG&XBJA#C$p9{UJv{zNqJ?lLtf@crMgVgBv{q8IAEl9 zU8a1JVgu0rvb4ZgH@!hq#jVO>cTzx@&1-=$aWWW{;p!5ZdBN6(69h~-N>@{N->3Hq zspZ|2M)EEq{4kyHlCwpVFj**8u=# z#!8$NohQU6GjR2jeqKQaI~Dc)ZLp+mx(qoBo-eG!;cMpWCYJfJXO?#}~}yl~}u^16@U! zjUEr!-6YlP)6N)J&+;w9L&`_|h`KCLLc)kJ(Ign+*o%?S30P^{M>1>vVr}J#oFf1U zkOxSSv!J{)W4@$+gJEWyl;c|B>2eMr;c#1V-{hSSW;fu+o*96($R$3)T8qXzv!4|~ z>^A=abX+v+yS9QQg=iK4PnZ&zjAs1+eQrYb#*F_RPJSObi^}o)(JspqL5sBpOx6&L z2dO60dr@9y$nn`7L8M+6ZL!ZIJG=T?`#B5ftK^)ZXr}7JTH(jy4P=W&kgZD!it#(i zHxgc~tFA3zUh6c&%<(fCmG8mGUL03%e1W9vPO*HimTz$B=7dsGvGm3Kqs-5?9s&lk z`GnFPQ-z)K(x>!!ZjqlWu3$Wx{h;shgjGE>l-sWZ5hQJjidoT&up`gWPK(s z5b^D_mhI9aFsj)1%oFrcVAH7n3z{GBF9Bidr@vuu^_yPMO_l?uBx0?V0OLIBeOVjV zPG{9#VHRcVZe|zj-9q2G5g5AV_g!_$CuH`W0+SGIZANM0Np?&Fdb+1PMnD39`A$M1$`SYrOG_-~~2GfUjpUJK#vLRuousc*uYN=NH%+YjPjy zOixq$dy?jqvuv-UPo*z&=XvQf2SSelbx)`M2*ZBesU(dQv7UCie8?Il_1q4Ns!dPF zI*V3iRp;j|H(SS`JsYk(da_Q;&@}on-Ow;>mRvS!JPk^xiTXH@B{?{+YJo zX`SYWF*}W%5vA!>^C4%~J8cu&gmXzuU@6p3_|;WnbrzKpb#5nk&*Ocv$_fUO+X~KQ z01#oWfs12{+qu<-3p)ogJAL+YV4$a`f9%4_{rr`dtl*uXaG@{8cGdGLdR#JLA|_Xm3GOQ2LzsJr_P`aZO)MNxl$q;5n)8c4aQ-kdVbQ=$0e|PR(Z&+M7Y6$tx5Oct=R$>POhg$_ zj{zbaJp(ghCMJ;8VuQ;{zmyc0cY}TYM(GX+<`L=PMcs^I`kTRG<^rg_V&_<9+AdvZ z1$e9%ww?b@?cbQ?3A)g~ujuXHAb`PW16?2Omelg`Oa-ibx5Ib4Y&pcD`wW=b0K@@| zo5X&zmbhf*)Mj!USh6A&^A^nk+(Z~LA3+Y?a)vh8?No6ArrQo>_-1M-1bD91?;IZ8 zN>6Q_AjKegd}$Lv{NS>gs(5xK=o}$)iP{l?rdvAqFp<2!y>qD1!t?N`ets2`m6OIi z)-ggFl^1r7@;Ko4b!;;3GtW(ioOW=y&A`!|B>S?R-Iu|)gpzn@%%o$$%pWpR39nOX zqrSiTGU1+gh6y5Z&G5m=^Qa6%QV5Vl?c#u?Fq^nz4d*p`{NC)$b4-X8bN#jDv9bJ} zrdg|@9ga2=wDV=|f4`^Ft9axOc_7F&VwuL)(#gkVsAW&Z&3rmq2Q-A-ZrLa~h0G`G1DqzA=oJnG)XH-K8VRRMheCjEcsQeRzB=+(b{;va?l z_KE-6(l41@0ow!iVqpnl7SBGHvn(u30scFethO5gP41Wvl;nEl%EN!ND6q7M^WR<_ zb|-DpNL$02i`#iG{x&f~I`8Ami)+NYyQ=iV^sfwVuao@is`S?Xe8KvZ)nlwiB!b4SeaP7YRc~Z2CP+ zSGM|9jKTA`N2H6<0a%+5cQRc+ggbDXG&1p1bo=!uW{@Nkw_5p;WwBxjftHLL)g$x- z;3Iz-o@1-Qa;vBHKsW>YpfH$gn7(P}&co&tWt);sB} zoGz6ApyhiE@qXZtAFrTqY>Ac1&wMu?gaBzw=2eDePRIE=Z@g=8B*?hNEsZ7Ni~9n} zbMSC9x%Yz&wN!uQ&q4sD$H~p4^%}7@7Ha?7=FoW5`l77u#+sP7DxFn8D*&AI%- zHHOP5xnNRM@LX-74U!NS(uDpD^^~9G5I3KMu`8qyUHan#M}pD%mLNXnjf*h5!9PDk zW$TN5Xx6YNBhEj%_Xt{pAqOYOKTSSVJ7}anB)%U$@)^1&c}5e5>eY{gxV5CFUCvP? zO|da=4Yhp;)kQiJ!W|a9oN&qJ{Cicaj~#~yxjH9b#MoIXt!S7#uPlAAhrgVxmZ?x? z;E?m>QH;5WbNKm&*@vd%o?{s2_-uiaHRT3xKKfbuO{ddknCg?D2C1_AuQ|BPmO|DT z>7!pF5#MvR^F~Hi1BUG7k$obzE_S$UA?1;G-S!g=^XASXvT+aX5XChoJhw&L$bF_9 zxZ+md(~iq#ludT}r5K$R!)}c!bcn@5bqRz*7G*a-Viw&{97Vnm{gkFLT14Au-i({P z8K@9xag3ZjRIKTI$I>&DT&9P)I_K_s7BhnNLU~XX@y=*B`NqeXRMV~63PK-torWpR zGbJ%3fa*EAvM%lX1uENK>?3@BQkZOZtqgdrvS9Zs(I*#oaIbm9@Bpin+Z%c@e zezTrji5(uW_yQ#fV(eW>f^=&N2|3Fs#kenYk75By|}Z390l6LwUc{5SU?vFJ4r1K-%;(G|%Hfr0u*PgrT@q+hfMD zsYR{)DhiKjFe^%_oq$I6B2-;A8Mf;Ti+!T0lbur07RT1sCSBXc2b|h57RT~(-2G!e z6g`d{@vJcQETeY%{a}1+zK+2Jp>Gdm*OY#77+Ne6VmkIZlh|5t0bBiZ?=%Y$8?;n|fU3bO!AmoU3C>mcA0SIRL&~oChoES8UPH_C+6xFSo%Es#Y#c*PiqWb+_}~ zvs0>tGy0{R;E{I4vUWz<=Zr2Kc;I*OiD2zh<&E5)pAM3;l-cMa6DJVuiB+{~n2oa$ zKjuS-MJ>)r@{WnknL6%4bCR~ivLBFY%0BbU{?VQIXLh73HDF?5+)c$O+c&u(#O;!F z?Li&bpp6ESbaZcn6s9qq?Q1hGU}QUwVR>}3G`}iEw#_~%n&wT3_-11L%|N`AE56Jc zJt9Nky6Q|~grk54&yT_MKw9o?&*+}&;lz<|13y)TOopWEu2Lnic& zo0CJcp0{-s3>HWbHjSwD!)x^3a1?Q+)P(J4cQau=x^O?t{o;5)X~;;`H)lw}2VW)A zb-P+nFLd6${2?Z9qc6`BJ#V9Fl#G zU6zlxYvjk(3rEaRU~l8urzE-k&z2l>%?sWrvopa*4G^~+Bspw`C^yDC&y~_J-8i)k z3-WD;bb+Bf%lUOA44`CM~lP46E=tBxXlOL0|XagxheWivke|=m6&z^fQvzUiggM;NZvKb zaW|mPr`fIB1q3asI2LIJ44Gd3SXK}r?Yoc zFe-%}whMf8EFn>DDlE@-tWs6jX&}40Dxm(AOPlHZh2g7gOrW15e~_Vtu}w zhve6Csn4B_j3$iC3_}<<=qiEYMr52^BUOkV7QY^q&Q4io4nGwYc7uua-NF%%(q6Dc zg`;u^#HT2eJ;@2GghV?F$19BlHN!hSgQ-a_0oA6uFhWaT_tEr74pvC(tC7vgu=|@e z-n#Ck+VT0NgVKWfF#*o_tt>HTVvA?zKpo`7r%s1xPj*S#BK&9ixDAPY03(4l8GnN$ zxLLobH8=~Ku8ht|EQmqYgi?F!J2lK|#~-!!DgPxg$D55E?i7`GdR@?OnAbS}A(|!X z9>8q=R%>VR_v*G5*>pzaQf%i%_5;r-SXHWG{JJ6y6BDRtJ29K3W8~!X=|Qbfo?@$8 zOkwv_T9sU(lpNB$s=}QIxcRw>@iWHdiF*j8OiqM!Bd~*=4ZfxfJ6hjFDAioAS5AvE z*cRP7RaUqhm_uhw(TG(Jt`ctP3%$%iX>=^n6~fyRVDQmmpRm@upg;cva*%~C_}NLs z9TVRQ8;FBFxFWVZv)3gdWgxh;w!|8Z%Bcy|L`rHFb8gAm8c`mUB+Q=lq5W_;pOe6z ze{!Khd{0tGk>ubihq-kElFIi@=BRm|qX|8_e|^&Qiy8#yF+7y)$lrUpFwkYO!c}2J zl38ol8O)ME^Rj(r0(FHLQBF^;RY#y&?HCuU73^1r>Gb%(1J90&wywNAc?pp{AH-wT zb+Jp{Ablaqdg73g)&D%%_w6~Q0cm_n{6K|Sm0^PKrvh2Fxxz_iGE56$Fh_k4Wdp~D=_j!KM z*O|LTLQ*XXKm6#Mqvj14)<-jvkCq+_)nbD_A-AeeQEBC z5tnPdQ8H?1!i(n04ru#_7CpUnuyf#52n@MSFBeu&44U(4Db!09&GMhCE{iE7C;L%D zZrT|+xyt1GZ{!}~ODT4j3fzpNv~q>MJhq>TOO%3oDX9Z4q<+D3=I9Jker#kB=hh zxSoJCh+0@C$ml-9j`=t*`M1vc$9^j6FYNAIS;kx^Z4-Ox;LlKJ`@D;_SU z%g+fdSux|Ky98Q4lVPKpvq}$MG#0O3y8Ic+i0o1vEM^^%Rpz$#Q^H|py1L{hT>5AW zH>&uYr_(C)YM%z{bP=(}wX@Fle$=SVfueqYDH_0eUP(K3rycTa#oX1q#MvjA|MRAv zkrT2}SrJVbG;=$vR8X+rRJ^47`ZJVjR~%AIu&lKEp~ZQz#Mp%rhwvVTD=7gr=I`}~ zKq79zsJ1`Kgf1x#K@@gxS)&7kv{(=(LD#2dS}Z<8fuosF@2sVIqKdt2atEd4D@-ls zy|gzSfbEoBM}?zQA;q{z`O28n3aGN3Vle1lnJGEoUQ+VN6kzDc%Y6GgMh|SGKQH7wH6~s%^i+`AQy*HP`EW($n+kB{|U!cwzjRoZjTtWU} zqwNFNkJ8Pbx=&o-j$rZsCCbM_y_5Xpr8ny&RR}Gw^8H8Oli20?_%n#?zl3F9>ppXQ zXjZ<$Vz=Tw;$E9d?9BZ}+g!%@LkMFRnt$dUfn2I~$7p*vv^KnC(u%JgI|n97MV*=nKmQDUd3L{Kv46~VYK_KR|C!wJ)}xmb!O=P;;Z!88S3X3zuo0YZiS?HUdQq@3s>eH47YWz`$)esWwj2G^s(lS)zgk)A7P`+%Qob zQt7wweDbl{-D)M-xhG7z>195jx%WJ;(3H0u{+flH%(@?syQ*CjqG^pDN~TC2_r7QE zA6BR*?{|rhaw#aiURxx&lxn6T?c_loFY5i#Qck_JZ_UrMrB$K_VG%=H;*^w#PsCJ< zABFS~0?;2g##wCG2?|P?>mh;jHUVao@sA^oM23c&TugpZC&YtBl2&&OAjtl*Eg?OKW?fHdsndF7Z_?H)TiQZTjR2+G^j9Xf&WEy*VYGW5W{h*Z1X9)M*MVZ#2l{$uPA0AF(-z;VKNn^S=+jHB~KDY zL>;S9{lst0{Fano)sJD_ayY<3nNV}XDv$pmT}z2wHMnZ8*?xtDVLO9P5~4-Q=RMT{ zCf9A0@dN1_CWF|UP$Pb&0$m9hMb*$*X(Hz!vq2ogj*+~)UAt#Y6S=T%#Gb)QRxi^; z(cB)sNIOG$RjwF|yAp(3=LV4%fl~cru)OgjwpbFAPV1}uvr;>Kl@(C>gIP&$ZxC3C zUZpd9F=_oVzV*w{arfFo6OFc1(M4xF>WmKua0vH4w1lVP^rQw&R+}oXMBwX^%LKr)KQXh5i$J_sJ&fWxNq?O_#j=%HZcm zjr*50Usez}>zfAQ``C+y7lIas#$^$c=jpLYz6)CSf<1wCd^Q{m zk(h0_)9-TK4Y`~1p88{Bg7rOWV**c*SLelI&1e3)u#G}pXQqi;+jbD6>|}w~ROj+G zy6U8auRF>t4Dp4Hwq}MM{pezPi37hGS4Ds`WEd^@Y92v-BvqwMcxpg1+{Vi)H(8pn z%%bj|zI@e-z>kcX@Dya^Eq}#_yxexBl>%dC%rHgm$9OJnzSet@7~nhI`gP$gq1AM= z*!viy%&xb9abqFb3z^aLjJ@{!qGxwi3`)^C=h^u|{l+?)Ox~?H=CKsWaM&fOR^0_X z_OViWpT4_lnpaw_lwgn3@#XB^Qsec_^_R^=*8*2|EUM5Xtx#eyb&_VQm1ygd$Ak-+${Yuthu-B54S7jd*T{L6urLZl3=C1X|S%f-BdtM%y zI}Fqn_AetU8tLqbfv|9*D|KlQwbX$c(NuVvw?rLTiDyUKF=Wzz_CXIj$3KzFTTs9i zcDr!fNxy^DM7BbeKCrc$0-k8xQNCipB*@I+gHQ*4%X4KALkyWcNL4ee>idIyzMO3# z!jM>ri0%C46O=@)ja%xqu-8F+qWrWRIj;#2JS?%ni~U?2zffJ%ag~Iiu^m8qByD#LaFS z=w~Tk8xf2`BTtNZrAeuosGQsielac`@IVIis+3Yl>O$U-q09Ufma8NMrl{o>F(^`A+(=;y64h5W4`W8ZN6=1*3N6=yn%f;O)QK1Yr4J7k}*qCAtFtW1g4(v-ehsCK$D&mMIb+3clkK zzrdKo-aml<;QJ3ifUV7ofFhZy`Ul_tcWQ2x`mMkJAkxjW{{MS!{)_eiz?RPc(iRXS zznthd+5eTlIA9$A6YzgqTmMD=cKqdcX#R&&*6-Od@&DTF|4-KZ$v0A2&G^jlC#1%& zH(0Ly#6=yw$|2Ii#l3S2X|k{F*+SI;z!lo?y^5EyIZ4G>jtnn`ni_=qhFeOHu%#)gu{mD+oY!>%&Q#(;i&vqh25^FnJsFVz|XDW zl1YFIL#!A5u4)8Jl#;4f*H5-k@)vGtegGT^d`B_|(Ehbkgw6IuajwYPMIJN`Pp8;i zvnvq)4YS<03sZrtsb*ZIHNjfMZ1s%N7I61kwCE;(tu}4qQ~}XPeLG?U=uMkyci4LG z8d0PUGbgV42=RT#O&vCher#4V=6i7WgO6D42(~mbyh-=2P@NhpJ^~qj%8BUI25^Ky zZdPg=-cWsv&ni?NKQ%7|aQf_KdB|b2q2bW&lUCu_rZRy673*-S^zMs3xL58B$S~t$Ue2)RN%UIVKeF7Oi(0^O1o)x-=jTXLfT8R2njmpE3$ThSvr%(& zUOM+AjGHZogB{o-v(K|k7aO=R46L%V6FXOJqU5Xf^n&kOJp%YLU7Q=toD)nSC71|ER` ze>bic#&0+Dx6h@;E^)#;;p3+zw)&{6j%LQQ*-ikP13zA=nAQ00o2>o}l@__9vtYso z2M~p+o&L11tHxW9BXlOcoK;1z%iafAbv_M=p7c(o&!;L#i}h+6y*v$zd0b0v1!B1V z$Psl<7XUn`Oxa;0^iN>=8-1&CFYO#$ET;ts^E8Bgd{;U%XvR~*>w28N$sU<}EwLpN zZeXR}1>AY8(5p4}8Ml7|Ex&E_vPC5ikkmst-|Pn*fPufepWP!7nIjSfTqyPcGj$qe z@qj;2N_7f|+m7JSDKo+I5L!dfGZtnxiQ*l~dz$>76$b?UWfd!6YcMm=vCK$r>)%N+ zERP+`_n%Sn*YY+EM@>y1$WT0?IsKg}M$Nn8%78T2m;TZB#$)bMrB0Yf0Qdircf8N( z05_MgRnX#0lIj$2njk86R3pw9L%RjwwfaLV*@hT5@uIBXNJcD5Df)8$x4E{r_9lUV z9wNUsIJds;ZeEH%ti%3c@S0%5+bNjxyW)$b!E0P+=0#MQ0rj@x=rq%)zWKL0Uyn}# zVk4|icb`K5Jjo`P_}IZ;f|U1Pj_jO0gLp`UI^a8lflUvlrGB*3d_Lm1;psB`^uu-F zx^S4Z)nX!OFe~mvQnglln zE(?a?PQ&6Y!am6!NhoAx7*X~m*#e@3CB46ltL$7I?+%2#b~+E>!JTGvTee$`wupMw z>%f8-JN*>0q@vv?$gBir>F@m3n1UZU=#ek6_4s&);7qI7x%q^heC7xT@WGVtQEGV! z*#6lj`O58k+E2Nx!6zMkF1v=K!pz zQR#-h+57Kie_j6!~ar1tPk>z9(7mb5uKsX88`7E0pOye}dF zIH^Y9xwnBi(m}7-vBr0eUfLK3BLls*-)xBj)Yz=S*Us_QdtPz*C?m}gn*7QLy3<+L(_?hqYNe z_-3fpMQgB@HIIb8fxz>G1;H-hreA&f`j3|lA>#SD;v&}?D!W$4Ia?);!KMMG%LUpF zs7@Yd5iE>6$2g4`>z;Qk+8BX4P(Gv%g>u$9rUTit{=V|hX=m=>Ra8gm-1Tc@er{nP zKTT>l0OsMdTNe}k;y={rP$Xgv=30UY@70Md(KPM=*bEzgE<&|Q9CxTVa^KTcVygm9 z|DMne7r-&UDG0V+(N}i}yL$#PR#Z#5Usv|{JCJ$k<9_R%z*f$$8sc+3Oa)Fj_0d~lVDZ`A**JwYsuEN+)0xssg*fu=Omc~8yka?dy`r*pUlvTRJ zz?xVpBy$0W!WwEnH8wW}!kz8o{=JC3P+^PlzHs0mzhnIQv9E~0UY0A83@KpetFde1 zT36zGpM0;`YWkJZX4>X-mvkgYFW?8xrwt7n1)-4nNxK8UC%Glj!FbVEtT8}dg0Lu1 z55H||`!@Ly$k$;P^)mQVt4$WRf)>ctcAUK$z+Q3jORyuKs*2A5cNj$!pCt_^dGiW3 zpc2d*ODRFev;g~=RrjkVi?mlt0<7lr3kV#r0hGIn2MDrB!LO$WD^3<2fq0_F9)5qP zA2jjs`?J7+>WW&rIY3Z=#S*Uem#bYutryh6k;g5nk-OKe^=%GIwl5jzEI8*%R%h5& zSkhh+#@5A>k6DA=W?@E5Nk^YFbbk^zL55b1nt>Nb=y4{vr@Bk-kGB7yK zO;RQIfFai2M}VDn*JUKjuH5z2Nk|3QslWOvwpd2i^kzh1tnA~IM(*-_!G0$>AS1pj z4?KtlLcaR&pO7np4SCOvE7C>8E+(&ETV9{3+yVl({sla&yXA~}zFxR2N9fOf(mP+n zFG@jkFV%r6H!HrBh}B4VaSi)iI?#**5L2hdvcNc--~|I*LhXzpr`tRHaNNrq_s35u z&i9DH+zX6Hdf09Ox~JNYPc+8%|L{Nbu-q$rCU?eeSbu&uo!mA0Z4b8-a z=n)do1m{nQ|1@<*^exH0bG|Bg0ExCgUup()1oeM@ZR$*P9HJ)3kI^IlaLHUav^3U? zTs`BEA?lEE7X%WXEy+LDJ1YJe>QPb0zC+|Os-?TgkArmPwOEu0=hH^+TtiEp%8biX zudV@QX;yr8rUA6}SrW#cpHj%$(qC+5xfXP$Zt{vaV1#T}*Uyz>`=cS_r&hLsKQ#s$ zS^hbHj0bQru-!^DA-_nlGx1n&a0vPQ5y+=N(|D>tsPmY=P(NT^vl=h`rbHeynEs2i zS7j8Vlw^_#*x*Fu(~-C}z+<`)3AIAt69AuwO%c>)aUW_tphCl-y{zTFaaZI|0`Sp` zsP62iqp^&bh=@I)KEq|hG{RU3p9^rx{;Z7lnWBwVdc3kRT+JxYRRr8SIm#i;HNiwW z(c=`aOg=PR4YhD6bIr8U4vnSEf6Q6i12WPR2`aT~x>}Na2U*C;pD9JT;b0L@>U1T;N;)bK~M9Alo0RRu%OdfWEZzm{~KG&y58ihYiP>S+TZ z3yCAgXCOvL+tiGwAI|!pBfCG`$Zx*^bm8YtRFoO=C4S#4oYcsjoS9d0a20wSG+OLx zjHz6<3!nP#5sn%Sj++-H3aXfHSu~OiYtI68%ai7VG0iCUl{{z$Sf(Pmt~Vz!Y|CID z$SiWT1+%hb5l(3qyQ_Noq_)vf;k!rJ3sTV48$#!#4g$HS$aWSZ$al&FHA$>%))M+H(yyyDlg5UzI-=!Qj zK9OJMa5eYDmYlc+DFvLR&A!}gA(8xKbQ$1Ru4ZX@`XY-&$o5?SlKv<3v-x|e3Ndw` z?%)A68}ov4dS{4Xe+{ND9O}Kgx6L^-McVrssD8nvGAu8AaJ(eN)Q%qgiWK=r z@*!Y*2stROtVD5Q-XJ^)NR;82HZEI=(fbU5YC(%b;qEB&buajFWKJ%8+o#H zP4GZD?-ox%3QO@;weTu%!ClScd4IauchDn_P!H3wl3Q!?);#LKWm_fPyy*@&IcV8_ zbOShRFAz2gb)G#y6*f5p{3TWJ1?eDG4+&sP^%$f|?AYg-_+U>U5sRz=lZ37v_5UL8 zy`!3Z-nYRZpooe}5s)Te0|E$$0#Yo1LJ%YndXW~SH|bRck*<^=B1FXi354ET5Gm3X z>0K}&p+vwCdUo*hEx&#D?0a_q+jI8t7f+sma%I6?57KzS^u{`JB5PHisUWh5y-59! z?qJVNW2&BYHNnWC`Q9Ojd-;s!ld74AF0=T|nu&qKjqe}r(!n|sWso4?Ezn)~-rRM{ zDd4Kz^ZQC?tF)WojmkosWkjJZ>6pr6L^?=S{K-~ySI_T3(#ck0^dW)hC&J%0=?90` z?5yj}icT{?dTyodS-EhH+n=rC1Et@qN@BXqxY3seB}O<2ob_BXMnHzs0wL5irH+)S zMl5K3Cuq{RRx7So0n%tABBPkUo!F=5>|Pp<3A2g%s#kr=rJH(pz&@!Ge-muek6Z!c z9CdX+>`s*W#Hi%C&NbNraJPxY1y)pRvE4e)R^(nklmiN;1CM+H?`C{1lh^8cl??|w;>2+R7817jgB5$pOuOG zdL6*QYUA?sC=j#l%!!&u{rIh2_EJ4Ho3HklAP?J77WcREPg|;=z4cDbh*|CkMQtY! z#Tr8Kzk7}s+_n->t<)M>!KqHeLHV40=W_3gBbLZRkQNj%v=fSI8g4vpB{I#Ma;)z| zVaeQ$Jko~{D-V$F_~39x;F@9K$vx7u>}(!)4ba`Tk_*We*~y&uj)aZvj6XmkjMiHo zZjC<_Ir~r?)WP~!`=mb4k;tX|$jpF^@#xDC$aUAwwJVfV?>DZEn7&Q=o)#3h&`haO z(DQZUMWXu!SgL9-i6J^)?SB5klG0 zgz!P8qmv!RgazHe?Fb;}u1gxCDK#77F5gj7m)Fo7$dNdhM;8wUuj<-2;iz7n(y4^4 zB}RxlqvouO!Z|xI_Ud5lQf*y+*IUHj1<^@Q+hX2$Z3r`@PN#vX7C*}YTfC%<5>ENAXk!wJZAbB8CiR4+oahCWDr_GFcM z;h{fzdhK_sL|u8}ZVDfPM4nbkeckPGv3d6(o8@5|8uP?U2+H`ITdW$m#aEtoGy1rR zL@~(yv!RXUf#pAY$7Iml=8Q)nJ82nennv7m2d4>KPIgk1s%eiL4agusx~tP_uM7!O z!q-ZJD6>F|feY9ZSB)+Sh3{X*ET@@A=^J;ahSMzK9!k55fNh%A4xD7^C+zF~Mw}!B zBGzq@W&|e{|B2S6Hv$H8{DWcX{rd7h7}qofi%ovWcBoD&((P&tP(M(1E+vXZ=#;GN zY(GdB#gg}8c_BC3mQtd~Q%ZsSm9o~5MKEoZ63EKeM~Q7IiD&&GS3+;48-EOgU4$|g z-DmhYlz&QC|0LwM=J~Jo1zKf;QjPmp_aVwV#}x*;0%cw5PSZ~hs{R&{Fi8J3u-SC1 z$`5pRCkF9Q$d>RY7#-6O7LI4)%0|T4f^RXpbDa%^psDRk*|U<*g!KYkR+Xr zO`b_Sv>98(-Z&d1k&#C1x7UMA3qi#zsibG!W7%IB-Ab`QsnD^d7tF8-v_94C(3nT$ZM@qLR3OSk4l3a3}rf{g!!abReA^=joq& z+aQ=b0()DxOg{6sXwSUY;^nKDsY8&rC%#wGn}M(|>{%#Fh||?Oce|#oK0^Gu2ne#0 z|3>$Z*cZOOmRI17cwiaiB>%3_M@%y%x5Zb<>PI1dSEY!=-_ZerB4MQ;R*$&*UAQ{1 zqs1%ZzbSM!3~zmry!qkLvUFiq=KXQBt-iPF zWc7bl!LL2F(#ackSKca?I)%YamD;gd>tFcL+*erjY550mFeV zErPP9Yg;7SQRlEWuh==uWH8YiRN`b`(CUc1#LZuOu0gN(PSoD?9c3vkayQ6B12)lhuj z^^42Cp)iE`XZ7~4rYS1x0a}nI#49%*jj%DcP~nQ6AZvr$VZF);0yx7Np~GP^dtHi~ zJ$fg3A%5L!u|iZDxpQ0PjpZQ7=zK|ug3MpYKBsJJB)Za@dx%#%$YJ7*9f{Z*@~}b$ zVp7rdd<`VvX&XtQ`zvIb00??P9Dy@DTPH_0m$PpE6G& z^A3v~SMh1uu@@VED0NAL#6y-ixm*3@^u}ZRi)Iawl|ZBl-KN%`GKwaoz%bm{GQ?+J zr=pIK&U*;r`UQn~Gf5lVn<`)8Z=}={(nGkWAAj(#q~LteEp)EE4V-tHQ`S(k^qb8n z?z_;q6Yl#K4Jw54-oQEyA&@TCK2Ck@Bd7fMo-zH~oIBzz_9p&ep95a&l8E!af+y2u z$q?7x_o(4`<(IvijrcKN5#=9xVe72qPN^(X13D$bif;P((#{V-*3oRQBXT-{l7DM> z7Co}PQ6F7zR`&2Jo$KA8Ew9`n{RQIdTW6~5cAzbs>Pf$&N`^h0df6# zr}>BS&DTAA)8k9E>}bgc5EkKvSovm-c#6GS7W1^%o-(J}YZ2ZmZ&MXCVFzSqvX1=T z{4nJC;#glkSPxwP_0{lVO$Xm}*W^5TrJ-o3Y)vwB>hhr_$%Y5=e`3$6b4>g2t~vVO z5(zUze*^h>A1m%gVsvj=m?LT4Lufm-oaahUm?Z~ zzYhOS3+Z&eydMGyvR-6FB6;cLBqR3=z8`-IaX%(-jwP)s#FJLcrWRJJXdDzctM$Z< zM=9*Hg6+c&)?rAcVAK60)s4%2sp!z-UXf+h22oRM8zV?HpoXf!I4O zrX(Vn8RYuxf73H&Lqkuv3!~>juH()F=4rV>i-&Vf(qS_D=-}ZTwE_ z2a7w8naELhi;A9YdlSE~^xgF0p%Nj&rkKEfl`g|0(=p+MG+6A}#;s_w*vVpC{IPxz z7PUlWh_TVRJ`=k@*P|2UU*Bu*@yQ@;?C<8*Q{YHSk!M(e$Kc+mpFX#LjY|#k`_?Vo z+36;{d2}yp5G1GA?O_MgAu|GXWraZ4o~cMB*le(c$y@GBf0pbv&;Pwt{q0Z{@2bCF zr~LxlV1O=6j0k+*m~@~uSEY;zB&Ss8At~c;Tn8C)T3&lVT(64}fUwXkc;1VGc8&QZ zge;_$0>%=E^iiX^dCL@dM)BaqjwB{>^t_x)$X8ID|;N%(k;hz z$;u!cq4<8-{t0mp9+YVN_~Wqq7LxGjbTp%1UGFcaoXDXn`YKoQw3BBJ%U`xdYnO**6hjV!%KorutHUqo(Qo%$(@;ipwf=>8mVO zl_f7Tx(BWDlY|(<7CAYY3m0lPUD+W8u?NHYy$>GrCKQt3=JK%P1)W*p$aH?giN<|( z*wwTMp6LL^{05ogUB~{vTW9JZ82+*p2{~~+Y_X(_9X%(|=x@aK+A~GCk+Lsd#4Fag z5NlxbRrpAi#A<^wb2}<}j7uC+B_v~6^Qel>EN3#63}lm&EHzYUO;u1oHoYC&P&QOa zCs(%u50X?#;^7Nh{By9Ig#2?jjAyk6m>%dw%)tf~9AX%7Xt>(C6(J3opZ{?zJ)$wW zcLg%!P!?W6ya~~!PV!iOWcRF z9%&q82ulembfb3f48>eSRh>!+z7fB|I8! zfx_{P{r_yeGEE0G!v(glK@zeWhUNChUO>iCMj?nPMo58{)3wuJJjaq!XS^a58i|j( z*?Br6xIkA{%TFrrCW6VB%}t&8E=gB4L13h4J=^C88>*d#7hC&Zq0A3eDak7?y3sY- zwLigDk-p_Z%o23$p9TXB&l{%W|FOQ2C;6rR2>p0L)k8t$gmi~fkb*`hy}QhsbmJRS z;f9utMo&*pOAkB|Arm1(f49FD#vqEB2fKptR4ZXR_pzCEzdvR|s!mryy0bo%x>U_@ z74jase3}NHNa!8aYDp!j(@!fjb4q_BoK8OErzn+vk0Cpj0U~%YtLW-Ah*{A+sT&^*ZRy8Z38HPKJ&r;@m?>80i!^DVZ2-AcCp03?vw6iV)pSHHxhgqekvPa*uvJ zWERP!Rez8!Q9slZIhP0T;1rm&Pl0#rlvABB$!ph{LcgTIH7EK7Ja#ld9b_>~GS#X6 zJow3C>7QD`lYhUq)n^9ID^P>MlMsCB$7J|qOntBD|7dyt<-yySYH-YiyaQy}+a3+j zklZIs8E9g&k|n29u(Y`>vM8qCIF-7nLkgt@(m?B2w$%L-N4D@ zv>w{7FF1X0e-IOifw)E>(~H1=M&M5b9QzFiXKFN!n5}>~@Fwu5DRKeIcn*q5JrBhs zv!Ym0;E5IZJNW1JL6PKx(%=DD8UFM1;FY7`tnUB&R}MbJOA%F{Ra&H zK?dZXYX1$3fAt!G3G07{{(pz6|0u)1 zl=L6<_!l1jZ2P)kHY>=@qZ^2SfKw4sQx{F|3<}sY{!2m>o1z7-!#F&enlK7 zffiS&+%K?OOE&j*7s|BsFIcUM(D;i~1+8Ezfo5r`NOJqhw5<2TI zXmZwe2(%sa;K`8R^b4oK9_$w@?9|>$r9}{BK6JZKwUb2BgK?koA(OdHtwS}9s)%8v4BzgbS)Sixjt0B($&R%I%!qj#i;r9%c*5hlj zJ55$1RG)*z0#UePxY7eGTY4MTm(hV-ci#(KfKvRdu(atss^mTqy=iE%yBhj4A#^2} zaw%j99lA>1bqf8_;A$-(wRw^@_h+xg1sz0O@uC`%(3{^v*M5YA z?xho?>d<}`9s=M7j}0#g9lVu8i{m8knv&}`dxmJUh%rBLie<>ArOJUU+gu{Ek6+B* zIhd$rOWF*2Z*_=NNLwb9Y%8zqcNkjMCJ?r-qnVBKbGZtoeQ1BsA?rWtrS~iOzZdR} z682^!_vQ*+M^%O%&?abG==#kq>i+m=qupZ3-QPK(dL42t!@E5+N_zaI1B~>eqNl&+ z+{X*-=o#|8eKBvC>7?jpmDrZ2vb#~pTp(hF6=McpSU5u1SlDserAmD^O5iZs=G^Zh zJno;*MN#WlA2h*JIbQ^-az*Tr5`HQH(!aEkY{E0Apx+2R1k0KplDZ9FH5*zgPEitL)wl_1pG7`{Ol}EZ#qsX{1Y8eO%({^~k#3l8O2&I^o#ChJ;l2A(1;5SN&G& zQkyIzSu|R_%w>8`UuI(tsJm&JT-3k(9JA&meMrQKoW)Y8s^+Kp#oBz>xqB*!i#)88 z!=l~dlNm78>b1)jx)!%k95tQ%wSQKBDNjzhKpct<{cQXkL%QBDGLqb|d6%}Y6FQ$0 z^5?F_u^c8CpYr3uJipz-{e{qt7Pq~06)#GuUpey1&V=P=b>wW}*lv*gH4mO2z<~{% zgx#0z*dJEeJhim7u0rLs*d>y8TVoPvk%Zt^R#>L?X0I3XG64l1>3R$~EF6Gb0B5*r z6uMy#=Fr7PJzHV!N^NE*GHQ1)z9-osY$pAHJbfU(1~g z^fnf);P?PX*;cA7>Q5!?5`XNhauPOMBFBOjO{c(NE^-L9MPUDjz@&ceg-o*B-k@8s z#2!Wvu_L87A5qBOq<_Z~;!IJF!ZF(IT&lR=yQPJ=k|etDs! ztD#bszFQ!vO<^EbleqO(G?x8S#UK|d_}W?Ypj_T;*+U^F<_|;l46U^l#tYvdZMl&TXJAmUo zR(8h;otDvIqwMnn+N77Y8YNXXz+J^(I%5QV0y5fiSw;Qu2e|Qzw-t4#w$jiIq-j+} z!kXDhK7%($VZkfRjgtGHPkzGcJ%1KY*}cm_m%AFl8Sa~yf%vBaTfV1fh zfBdVcUowcYH_$6}TW^RJ)j<(fSxd+Wx|ZkHpqq2tN$s9vca^Hq%24QRC1V8td!hU@ zZFlPoA}|udP!1V7;=6vSR6?iVN549zf#jlZpkHq+`w2l3rxkAyowbWIbaPnS)Rzho z^Vi%O99pq=S)WwMDQo+iF~iK&jYN;Q3B}4VlYWM2TkK0rT|~xrluDE%OTeYm@`_9R zfn|@Ue^v8-5PC^jn8LD!V@zfR=yC^H!&gf;D-|VjGW{0^#o|N3NknEC@rrJ^MSzTC z3b&K;_1VdXhuNH&p&J*P`?@{r<|3?MniOE}xDkMP!GN>j{2|7_!eLv*%ZCKa^rY?>t?&8~U?+ zHR_?Zle6_WuW1f`SYYUC=XiM}#(FM;^-1jVES@q9$FOjsq9L#@$Yb0)eU>}rNIr$OBK`2-hV=}xB7@F{hV6kume2aDZaGXx zT4D{Hkf&~_%8A9l8eGS>T>|$AJP^n?HOY-VvLZ5}Eg`2t?C2N~%->T;*e{JCu+hEU z_FY@ErsTsLA&8s&84eI0U&}B^sk{qIU z4cLl3pojhrozR=7L)uH_Dagyoa3Gsvsp z+9Egz>aUL3q_w@Z!gZZ}91U&+@Gk+|AYiqp8(KHnk zYsn)d?eYv#sc0LfsDCRG!!LX}w7~|~I1PP&iB-bsXfXhbb&1X!8!O=RM#1TR;{kOx=`#d- zdzKiN7gCo2{fE^0Ta(g^fic4kf2XOGv7ZmELR{rfr6#=EB$wc`f-~DxTuoP;9z&18 z_#~hy_DB<+IuR=jldRe=oNS&R3rO@@U9Ko(AvN3ld1_I7q7=rW?k5&c{*99YWJ zVzYr1hzVw`jB67eMn+|ae`WlUmUVlB-=rOF1$Quq>Nclo1|jya-`V_0eqEEwVmsyK1XKlJe5QCwPz{@$ zR;RK?iq?wF1QNE4;&;&U7iacoHi*>O&`zl;WHx>{d)vP?xWXcx=-*G~lgh-;4+c&Q zkm>A9Q=1|vn{C&UXT=?n`QW(Kc2U2`Gx(~}&N#J!5E^BLty47LFY)d6Yqlw<4(^&R z1GoNnL|h5T>c-u1WU6-_y}8{WUUtkccaM8E;05tCbt1cbHYAsN!0W>FDPz`YYw$B1 zL%O2Nz{?L@(&y3Z2x7IMp@H;u&wZhIaVuX&D zUHkgS$O>CkC|j!34Wg`vnwj1X-Bqp2);vQD`6OU=sHR=nh{#Nn(x>8#HT+K*C26#! zYy6%*1$TI1#R(fD*}TSxrl#fl@5q}r)<=X9L^hX?CRE7pW&m6I|Ne{&JW;tpSRyC zx1iisEyl}Knlc}cLp_QObrQF$a)RzuvVReMQv7YNdAKq@DZm*f=dekgCN}VQx5}0# z*duS7j>gyNp!iG6Ls6}v?2kKVQI_;DIxX^0&M}x;*vDNOrt!A^(KOYs@AoGxj6`pr zz}x-wq##*w0>Udf99*Lq;zGcusLhFQ70GFDNOp%!pM0pg?{u_kaF0@92kwn>ypEo~ zA*1*G(YQUzAzv&(BXa^{q@%fepVt)~9&S?s;|qEqUl` z-=b8-d&+h+y0@nc;dU2WvTMbky6tV@<`&|L-%Cc%7nmSD{+RSo`Ns4mTl{jT(~I

h8IgQM71^47}Dpx6^SXZ*!_u6%Hv#K&frKD%$--W(K~HS{1Pqotw(l@z;U zrl8%a9Pjd_w5_G#1-i}2`r4*`MV7vQ#+-);nUk_F(P^Q@Wf@Jmy}1>5q+U>Sx+qLB(h4?S?idJ^P; zQ*}dd6Wo6gS6BCNH<0D9TJFU|lJbuRm^YG*qt{CO@Ww%&k1mJgbg}m{ua!N$0XW{4 z^Wicn_BV0%7SXF;#zvSY`>DFfL|j|wM)llC4QZZ_M(C<4V_%Cw6NAk*bjEe|D<@f# zr3Sr!=PnTjgZrOg{od}r?VPQQ&k2H}jPquS*SuUKCP^NhJ9t5#UWoh~Vq``@6K^6<%qPQgx!c>27{Vtn`>}W=l@_af`GF97l~KqwkD8 z#f7Cl)?%_XU=G8TplE>KGmVVu(1KGo%WuQ*^Ck%zlbjv|e~mA>gG&$mN16g!i>jt} z-Ri$0V!iA6Rs)OqU{qcHJj&*lU&pO{zjzNzI}{dUcV)e*yve@_J`b4h=pURm;M%j` zOMLPBGoTje<sJXz&`NIAn17BDq$vhcY~WFxlK#IMiP>X}g(*!FFk;RAM-5 zqJz`y=c1P%P23$JuM#LrW?QT4&i42rbN~8&nOAY~uY&3%7&n0d!WQ)pa%0p5(YtOQ zq4M_<^El^TEH5$!d**)h6}ZqZm2}8eiMs6@_~aFyS}zMYK5m`tey1*%D|-y#w!21_ zR4NRBu6bO=4Wj|=oA{?xxx)c;m`(q14`U|?C%prSiywd+YYw9T(WQ;SAJ|#z@$;uq)_ZqXl^CPy(!nv`o#O4aDp2PV0sgf@9GbT2aG4Wr=eJZEN>eA0?CBq zNz(2PjG&jGvFj>%J|&Oy12%PM%*E`s$K%-MYZA*j6HTls6mr{Y8ugagxoZK->Q$F3 z#3J8%b{}4;Pz0Z5ttjL3zAQ;dg(^|2XlXX*TR&C`tgmL$J2t_Wi~5ysVRvgkkDP^b z2|r~6%{EjlHq|4pC zH>Lsg#98@FYL`^su@^*-PVy~)OQ0jA=hYr$T@UPjFM8yGiHztn;7lup5#2R27erL1 zh8q;W>o{qxIV}O=m2B_a_?pp}`{<`ylWv_XoqRB|UD%tHj^RJ&o#v$dQ zH6#C{HB83_=sp;H6^Wemydoy>4F44|p3S0I&VNaT9B`&I%|@>G>|j7_9UpA=jLo$- zj9Nyd-(Uc29lM`C{x$vW4W_LkJW2D4Z8zKloM{=LOXbyW|M8BOKPF6z<+3=Vet=bS zfO_FzzG<=Knu~fu61r0m$1?i|8PxIQeNg&f7i{$gMe0;ANGgf%+m7B*oYVwA=fWNZj=Z~&f<0g0l(Av*O zvd(!tB!UZXC=JzH`o@n)?svl_!8+diWj5#8X!L+pRNAyo_*K$aN?L@;r}&XDUAN~S zd7EBWqZN1b#>s%I&ASmVyCL6|WWN-BMU&-oS66mFE?}xW(>~U)9*-YeI)k(w%~@Oh z{zgfhTsXM@63r2j%gd2hKdbUv-Zq|FGB`pnjIJT=)F?QGM?Y;m6B%Ji4;zkzBM zcjf6_{z?b-2_yguQg2^iDTqmB`fp*GOjWCdBY6XEz2_}>Q3PW!{vfpb>W&edA6#~ zV0No*c|ACJ=ziLXyh(z`*xE3lB)@5-sP2zXo|RIW$9RYdb_HC^Q`i-OQjGnwG98f1 zeSs9RdXdo6bEm&P>W*ipSwxCjrwhG%b%&Q|Pgy$1scAhBRiR1mXftS=l!~>)ssSdl z@+K4chly>16@o}<(NZhov}R_8Xo$)Y4^nad09Ceh%0xzMdE+(0@uC^3QtGKzF`>gF ziE1oXltlICn~Yj@QX2YJ%$8v6W*BQat}}@;cr7lVVkeNg$JQ%Jp0#+0Q#R=6GV>T4 z^nf4Ivl}P5T$U+P8z{^ho{~R>u#hCmCQ*AIjgyRfNh4t4)FTNUFpV$h`pNeGeQ=ZB z=|3-~5~5$i7_vz3V7tCf$ajE}$@|5HBeA5HU6+HB?H5N_QKR>vR202TmK`lX<{Y23 z0oH&ir_$M)KY{mb^lY}bfP8KRJIYb~eHHuj&RKcp69c6R#IF{7uVdyXYZJEp3dlBd zT>MH2y1C*iIf}0J7x*GD@oXpswU;8jSG|7iNZxuqC~bXu=|$QLYo0;H%6}w?gaHX2 zX*4Ok>tnvt9ggu~E8Y3oIR0paFr00}$0Kzn%Q%lT{UAg_{%U_;e`Br-}Cd2CoP6 z?%9L-q2Lg{A*z(u@!kdtR6J(lrxq|<;|FAE@A}^q>GL^% zu}a|m^CbaI&XESoH))NhqI+%p&p zgY7(A*s5%OUyfeO*FK1TzV|`zIf)6t>eFI8H^mc-S^lU(E>9WmYO~yovq&Ib^LL&d zE@&&~n8eZFlM2BH*IlI_=^BoWPfD|?+TZ&Yc%Q{WJkC^ce$q1rP{T!HJYQ#V7Rc)x z;ljC1VClqZ12LwQXV>gTs_^fzHekx0jTDoX6I@ilNxUPmYwPU^N^BJl6UYU*&d&Mr zr`p8?l1(AS%r{9!#0AUCyD09G`b-7(ByW=@znNk`(jizjm~W1^TR}Cs`y(-fQbwpq zUcMv6d$7WRptt>ngfwd0HZ?HeRh!EKmq_J;_^b$2mQ&gU4Ey!h^0;sb(m4%D{nD#r zU6-MVBjEwPR>(6Sd6ABc;Tyn8+QetxRno?v)5uK8mAx_0=NW@1u2(Eo?#g9xf2hd) zd{f?b=vaFSW<$NyMNLk`&57kO@sx>^`bVZq<)~_&h|0CSvWn{~rOK@{TXw6%6sfNY zsKuQxZ5m1y^)`NUvMKh%XZeJQKE;3-?;Jq8ixG|?+obd3^4ZRBAlb^{N*-rti zKaQfUFq*IB!|PWaBa(gCWWXo6Ww$FQgT*6~S-kUm8JtN{F5>s)*gQMhdLIP_?1;O; zl$F(QRexJ@rOXn8rJ2oIigi&v`P6m5>B*Dr#F*zf>~Zai2(;yX1ZM5HoRhsRAe4=M z+12Gke5uYS$31bEW%18b-mH;mAzRk>iLW&-C8$Y4uLe%p9nKsKCi8$mYVqf^#%_Ms z#`oF;g^&Gj(wsz~2$T@CaxubiP0KLL#;UB|6vyI7LD4xgzO+Wif&GEsMtl#VZM;PO z$gn-{EPSF6#y&P~8ghj&xjsB-Bg2fPwAuly7`O1qd2Ms$Vk#N9Zxs+;bIQlM=Y!FH zo%s0@-q=mOvsMjw!^c;7&fG8n31Xm`;{MQ$6R?4?S9df|_eF&!vIZM{X=cl&q}|Fbj3_%k1=s4y2nqAVZ0JlgxXn19TX%Py|=sPiz*A^w+KBF*aBQX4zpoqvDLN9eARRcIGoy3Q(@t~sJ+oV3jvrcshB6Cc5 zP$q099aIE40Wf8fMxs%fcj|T4UKgajcf1CbeWT{|IH>>q2f#-fKrJ;jdhW4YO!zYl zE6mw`-P2~H^TdB8BpBVoVV-D`S5v3d`pc2F$LfM*FJx zq?-@2cnYV=${1`J+>wVgvtH`4FvVpqK01TvtszB3Baud=h*%`D%EW65MpI@!IR}}( z3@v9r1uQ}WK)b7BgeECW{iqVyEJK${oQ1+T6i#R7KCtii))*3s_|#RfGTg2sdtRwZ{7l{@bc!efcn%_sIs+Sq*63uf{uycdg#v z0V!3`spL{3l}6iGxp=q2cBC`pc$nd#sC}_-AMPZbdA#uT!>3bRO!N z-I)vq*`4G1uNL$#cI}u()akWj^Olm;8omQ=_F2lk$=V6-f@LA!ly^upvBouzatWrC zin)FBm&yA9>TWW$JW=j;&+vqoep`D}3skCAvvu>RF)qmp`JuxRpZNt&ZlI~6{NbOR zQO1Z;ohf_S4b39#00#Fic?ieWP~&*N*516Fx2q2^F{PL~VVoJG+mV14XD|3qJJZnUY_^gyuTQ@u-7h+7VZpg)6UfrACCL+3&ZNc)reGt6l?o=b*>qxk#dHb1jV^ie#9 z(i1Dy+M*7xFb&{a_oMS_Y8h!ay@l*lBhq!fFEyu_(AgVUkuHt>9;jAJ$uzgag9cBT z&seNxX2e#ozEBp2P3Q~bmX}>|?n{rL+%AA} ze1Ms;kw-La5nb)e5Vx_si7$$mZhwt}4lK{JNjV!jI9y9rRL`eqOLW;w_s(Z%-oI4R zuemuIT$p=%wDxued5BPUQ$a;xNPA|^5gnlkok88@ZReZerL`jrB5trgEDo#gAY z{L}SkI@+j8o~u??iKB(GBQ%RF$DCd5HM9eySUvUphKfHz{UCR!5m=1m>;3 zYI`vsJ!H5r=M{)Te}PxpD_U9-j_lb|q)OOBV8)_`S^g&7^Wf$7)5cou)I!!aoTK)6 z+(6e{Ql9n+d17|rpaV~XpjI$0w}w@j+hzTwx;?kaMSec|B{+X{*mew4MNvhg0qAK3;Di>fA zOJh16aDO-ZdAmVr*9({N=X1;i9jN(vq9(>Q9bVXhH$EIFq!U^#C#ie{aeT4ow?=xy zNLtU7it-5Vo8;JjkTI#ue2PsXvv8wE&ik2ms$&~4O4w>u^yR@c{K}?Z##^(f`JzJiZS4S zX7b!x&bQq z~G;$S!?}euLYkgm6{1-c)kwE|qlM7F)Hk(16&QPjcBbSf=?5CCE%< zq^G=l46r;%ih6N1i6deE`id$_!8je5jC|aHOL-aSSjfmmMoIrYed~bWcSJ+}OZ&ST zK$G2~Nk*kR4g0x$Y)>`xrK5M^ZQn0OK{?Y8Zjq2DtY&2C#$GkFh2~y(M94C1s^~*#3{`{WL| z@oVhbJ=gCWn#&NI<=F;dsT^Q!MQvLGXGULs2A@-0{_VL_Zd8iKLA`~()n6PyHE5t} z@X!<)r_WUVE9YUaFa{<1t|KX!T*1$(E$^VZdtNCRPrHw9$r#7Qnr~-{(`j5PE1Ww# zy)uard6!D`&4-2rIG)^wcTn3Au{FgdAF1tUoGJ&B3{r9%)KIve&o^O1|E(A&9|< z)dS3G9t1s)1z=IbU$b#LU8uG7ASWT7O}@fd#5ii-FrL>(1t_HY?$1)?Vk!}6;5jP? zdbPy@fj8-d3o69o;PQg};ad8p_Isknj)>rSy%ZN@3aVyj{)`hr6F~Gl7`bXu{S@C1 zZ@v~B8p^T7p0d`q%^$Fs%3rweYL_vPW|k3^^Dh-rO2H_9YxTGUo0xnCrEr7U9XT3x zzEb+wdwatzWSyj8Up0qHW(QpM#uGax^Goorz)}@ki1}aoSjZN&Xl_i>+o@n9WJ>4t zJQkw$`%k{GDb!f3PP&#jVdgnjZB|Xb5g!bhJL?-H9o*I4l^~0aw0MTkbCeo9J-# zuiN$)gzaqsd!hKJ%=4=@szbV^;QlC7kS(}G6oEqF4r=FU>eOWEJfxT3vXth-k>T@H z6ZO_-tv8=zJc(o@BadfWb!`jgEBmo?(U=q^cQw(jA?N82((x&S!oKHZG#Z-HlDlex zl5up(Pl0mub?OV{r&N1LODUTgH>QXD)nW*2?e~w8X(aW>snvN}Yti!^;PPF?K<|b=fD&p z?67{zSMw^6tNGvuoU*s44t42zN{VY+$X!(Q4nF+tC{oJyV^wyK;M@)vJE~}=%*MF} zi#$jCTzYR)fe^Pbi9k!LuiuJ&T++W7DGF<`pP7p-EVu*!K`gLr0J5A*!ON;p`=>e!#D`n5_pnsl87b zmQh8$HpIUuV^aedcU)|&*{5t-@4mK~hy>bP{k9^*mcU)lv(+ee~J5_pWDC& z;Y#}BpyjNQGv(iHi&uSIzQApY5w9nj0$@g&~t_1^iqh$KWro`%rQEf+l z88jFlwEg)~I;IA~eYl8R+;z;5~pt1W{w}0haC4TevE_W}hll(?ye4y}R|N8oU zV8rkL{7$CVVf;A3ZGj+-w);|!zXZrrcq#MYjsBUS+Xnul$}4Oo`d#zpQM3AgzNC~} zgqB_24S(k$y>SDWO+$mB+=3~`$=VIy+WB5Q#Bd&Y$rs@G>+ttJ1 zL3qK@55`=O&@I|jFU+!62KD}q@-mEyhEf87C9?jOGx|HD>|Slkb)4$s@6RPW)u)D^ zOU77A4Go-Cz3?Fizw0v(HC!}N`m7zB*M~@{(Pp)2ma%;`XYi$S-KhMphAzw#n0%cFFSt|Cr53?P4lkY#tW zrb3X1IBy!|cwiW#hc%^55ze?i1itmLo|c1@&pz$_tuw~B%uhK#WkUZd>%?N7 zR|Yi^^r0PIuFxN+pV$%$ebUc-&-_AGT0fka0Ne%YDRSmZfk}Tpsh7>zae&$VTF!S- z#47dsUdN(YTkq{*T!+DV3pBp%dOO@-0zsuh{n-o8r*w1o*xOS5+XGX`cTn3b z#yBTFAJb0`7vPUmJPp=*v=iIO-M7ER>9h8x=7~gftrr|Dc(kk9;X3v6xHsR~j716j zB_m4~2;0T;?`!z4uk|=4cp^&8TZ#p{ycbvRm*{8C-*7p$E`uMu2n(fnmEZWhRaek+ zZ=x+gproHH2DAHgp5e8il(*+sO;pEItryViF2vg*{)Q;rdb{-lSE;JJ(d4$HNA0im znx_4Vf~H{J8FOTR--s-ioy0@SM@tcmc_aMDVrZt50S0$5Qzgd7`~F%09_Ng(avLyM z>qTMK%(PXTljhCw-7jBTi?7R1X8bU({gP5=sFGzX@)E-m@!}C?W3;Sv2T{_meQnFO zro#lydez591^8%6hT-=Li5PuE3ATMbPzYw{V2>Ec%Pe%wS4=hahaL-E?X8+hDt&A+ zl53jxy>}eT?lYR`dOa1fwvRV)Z#7;KO3H|dcQ=38xw1{&1|vcSD2FiBGy#L zh<7Q_@;n0>Hoe>7FQT;sXFyPfur+{dTcm%-8TyO+l=K%c_u<|Q=mGDp<~<(NJHX#u zgA`c;L5{s$9sjyQ#=5ISC0M-rrP{qok5*SoO}P7*z15)i1p=6d4co@a7=}U z*uqX6Vy>I2SkiCt(Ww2ZE?8gR?0|QJ5D>~lEY3QscRgWOs}>b(e17{QltX$%)3al? zfFZkPrLsgnI251nT?#&bitGCxSC%A}hBuj6w#_j4K{B<)Dx?n7kViE#3+|hZAuiUw z!uT}V?sffKzlYl16NTC7&gz%sCL&UFJrOx6xp2FWqFLh|aKV!ZU0mKAdtaZ*{_(BI z>dY4+0LUqZ+7<$fFNd5jbn9y@tGee28JH(Qty)65aXc+UhSt3Rz!YbEbBRah?2`U_ z@q|*7^g4+x-XVD({H`{!-kegbQ_g=LR(}aL5PetB|E>OO=3BWP*JO-w2A<5iHiq37 z=Tb1)n_g$Car*3qXnlOgK&ZmjtO2-9ni~zDQ0SL^t>J*ULTbjDJAF z_nzM=Ip@3~_bF&I)VDJ76Tn+l71qu_?tWvg;dsGtHz5?(m87XW^9|uh`w^CIl3wJH z9{s^91)pkuH4Ub7_2?Mf{Abd@j+s_&|BK-p^;qe1{bqn&_Io+b^&HulT-#}oqT`y2 zpt22{QB<}Qp`=0^~*sk;4POqCGck#_PPt&U~KJ|*b--{c6hmy!m+V5Dy7ny_y~Xt zq9W5Zl__yuEC^03H}HTj`;5+oAE35~s_Uz`5n!Ct4kP{AW&K^j@ZCbkerY6(NK81%V!iOem0Cmt~z)2&hxd6L1$I*x*64IzsB z=LZNLKK$h1w$s$#o{lQkp zOn1ahIEf^hXBxArbT315Zj?0Fnwrpp_g=S!7IX*EUAA+L8q!f48O+&B9S@l(=I=X7 z_zMk7%>#2cI~r*E60K$zt|qqu7({=i(h|Dl)mu>g1UWfzq!PTv2>8HC(Av?31|jLz zPRt0a%PpN+#W_>jxv=F%u&jEp>WmcKZ_&wY71l!}EwE^^CQrf4G*;oEYCcu-ySpPw zFf(QUkb4N_01MaJFb87Z&5~2DV2C-;$mp zni-zf9YMv8ZvEWisIt9eTetw9yeI2v&N?h8c*yg4DKA}##EuNFkDyM_E~_Wf5LXOCtTy1Hwe-o{fAGG z=g{oMa7!ZXmA%ywR7>pmzI!gjkckKyybCd3r2K%xE}EL1U2Liusw+K%__}Y(jgg4Z zINvz<)3Ho*VnEGhLSeYkX6LG9H1&bc(I&m$+s5jYqKX^<+fgLa2z z12u7pV8k0~ps>0XsXrklAPajMmt8LJD~J$WR7|KE6-V3A^aL$%7QRqgrq9Ap;WWwv z>L*Y|7+(bqCJAxBO{$D@Dgj*xjm$_#CWtQLx6T>);GB%lE_CqP90gKenAZz?pCj@z zUQthVvlS@@rUpJ8;WSFlju3%9C%Dzq%3mSu)yb1hE!yq4SLnBJV!ZE`3u+!Ki;ou( z&WPXhsaO0BevP|^r-XvxO@G5cb!4u!5n*bWTdnA%_0uAmb96xl9`3-7{B5dg5jqN4-42V0%xiq-yL^oVk7^UbjDr`JSzOceDQvX!(P?9Y3Q)m15NVl2tjRwLc{!kY zy>yr*xdH3K!4?W2f3t_o^o_j%)!EP(OJWQ>S&ElF@cnB7G zzQWFz*Om=RQ#HkIW!-)%J*~!2g`na5NDrv@)b?OBJ4)C<(+q`+j-?ntP~@nCgA7w> zV&f7kRXaa3!vFHF=z~MZfXH`PbuxUI%Z-L^)OJ*|hi3fux z=W#t7-sY;?;_tk4>Eg($_4tu{By`FPem-LGnz8-)Q)wKTNHRTleO6Omam7c#N7L2I zmlQ!&bB@jKDDaqZDg!!B$o9b6cI%WlMljBsGN0}!NvTQ_&I%F8%}US&eKyt!`eG#HCSjoRd=F;8 zc-l!KVHf6Sa$$Md>iyld6HN=i(;@pXk+V79J_~@Fj*hqCAp9m&E2UK5VyQ|o6qC|~ z7jbLKwcHtB-FLODJAl+XY`csG(RxwQQOzsEYSUIzCW;*(Y{&`=+waIN;CcJBTGOh8!MJhmL&A3srQ8w%st#QoimgJPP`vJt!LeVi&uTp83le9C#cL(<#!dFHp2BWtzT==TH2Fc4IEIeq`k*T=PH^ zUM#K&)T%EHRH^TvK^Q$UvE|(omxM6?0pp&k^k$AO*mUtcNjNoxny3pIAIb%8=|(#r zZs;T%=6tCt zAK5Uz(}jfj41>zj)Jy-W+U_!I{|heWnJPQ+bGs$K$K*K|E$KjLG7*c2t{=65m)N5olAFV5GLWzx@p&&ybfKUpj*6GhRj~g4xA+6n4{I zQoeTk)%H5|!!N#iV!uJ}Dr46p{+le*P1#9OGh)2RzDH%cD#v$c0yg)VCg+ApCmk+R zQ1Vf$_qHB?O7aYyvbmms3*GLv$;m)Lqo)}8ZV`!^u-KA^h~Ox7`(U~!5G-F|3_F** z8B~u%D%PL^rS^aX>bC&*T;g>Kk%|mfxQ&6o)lh9Pt2&&m$owHg zLm5BkqM(QOcMjh-^FkQ2~y+RvVmzEZJ&S@*VK`<|;6=d0Oq`~cc* zJhvwfFR0367^4)uifqAN*}fqzD&QRg6ds*hdLK#A$`|xSxR$d5z5&XQ_haCh5XP^k zAqXsjJnUg6Q$aF%%8`D1XTo)r?@X|(_98M}CZc%Qi^I!;g(Bs6AL#*MMkB4On5H2h zQ<)~ODq3Ns_N+)I7j67YwK|4%V{WSFXZPHVwT1=Lb)t&~MG33(5B8ml_qXjY8v!$V zle|K0JE*J!9gOFtBuBOWv2`Y!y+i25w68?PX)UnTdmH9@^qL1j4ub~2dDVKpTG0-( zY@kAoGOE~__abOAJk*CHve2l6{H8IIV{g^(&9gY!=~8iut0wz(=WP;czAW70@B05vEVwn!!8w?e~d)ZqA70ujOYjka~$d8 z_f<`HScasynp<2u1SnxE_FY@{NGIOYxWHH*Odi$gkMm>cPS6q>!vss!@>R&^G%%|c z14jaMBq1%;+T`!C9bVZdL2B28Hzx#fCd|aO!(@7lR~h6s62Wd77@|CW?gSl693kEj zoGV4LN&R6&(b8ky6{ut+9Pj^|ea~6h0ZD&teRXt)1{a^mWBW|@olZ9DWk>&|2)8Gg zTwAVx5ITMnj%^$N0iV*v?oCpXq!tACko)84~F&E0Zx$Q92gH7v?o5kj& z>I|_VaaS`7x1U!3b)YH0tVacM4q6n>NCc=d*)YPgd@U$|-B&Llkb1!>myqpcpXNp`_YJA=34K-i0 z##2h0B0UB6+>YH9Owm+d4a-ARS9)qDj}P@FKn;>`rt^Eb%F)T}+*Zp1T(Gu99$i$G zEbSzpm{uCbOMYcUyH`N2q-t&~>oz;tTE;E8N40Ovu5^G_BuYZ3gd8pXY%evx8iRSz zKEkYvO&JHCVjcK#r!mDfuLXu7Ldu{AU7JC!MVg|mEk(uE?8s%ujo9)P=q&i>9W?)J z8%!|nS3bxp|EU1ig&6LO^Aj&t22{vO$Du+8;^G`=&(s|=cboj*iXmX*)S~ zU}o!^3HVqcqL|YV#*(Cz2ZHf^;Uk^gL7QtFP;utDtfYs;nBK}(u+&2COwdpBhK`Ix zrbxJCU;9_6 z-aYF1Q?__mQ>q`*qkuk+BZJH?lQ!vwg-1sM5V6sK}aoQ^ylil3Zuf=;4B=li_Y~;i3f!TijK==u+pv zXNt_g({X7*jJ`i!^c?G)o;VP6#^<@beZNmNGi!lSGQqDX^N4yG3T4xC$DTRnWoIV4 zDU$AsplW8iRtsuQjr_RuS|vYmXIPs0dWO3&hO9q7-M7uug3}YKvF8Y_zd>rkR1m2) zWWAGO6KlTKv3~RBR{{e`@SVZPFtwh^1;PQ+L*6-&rpibnZis78goEdD2tU?HpyNLC zi3BF5IDLUk=JRBue59miK{E+AjGEOKkSV==c{PC%UpGbia(8lPM-D(K=3v)1D}Kje zH|4xg?Y_XfF-k?5lp%3nvcCkqq-PeZHI`0D>A8VPG^}^MqS=sSh>VP`Iqy2yiz9Tr z)DEcVWQ*Nj*k9K}u|$cbGJ_(&j;p%%#i9G5Jt{LZ#b}rPb7t4uBnS2gr6ZHgMwqAy z45PDQVvaC}cI)a2{*K9@r1Z~$kwm(L$CY6B&#^`0>EU2d z#^!F^bSFn}Rlr7^Sq#|y*{&o%tULU8$N|o7_5U3?C4MxK#D=KYE-=2SGXbxs)iX8K zR8_r>uX{%lwVx@=h)t!oz~m>&x+&>?2qJi3q$>O=MC!b<=ph@{t)BN)H6d*2}mnPkq3e>g(5|Y(CR$-K|w6%XYj!yI2bW8n(f$98vca)70G$+r*_( zmtLJ+Q_;hw->S_eMKcl#A^~F^hL0qhzUAmJ1m@*-%+QzMNwC;QP zp5UUh} zE7u_YiTHmN__XqMs}M1oqsDa*t@q4<6@JkM@|(BK1j>>A^+q3@O1|uMUo?jN>Sw1a z|09&5oZruYsAjxd4t@X|H(V$gsgw>VryRfr>yh*3ZyupH1yF7rh` name, "credentialValue" -> getSecretValueResult.get.getSecretString)) + getSecretValueResult.get.getSecretString } else { - new DefaultAWSCredential(Map("credentialName" -> name, "credentialValue" -> getSecretValueResult.get.getSecretBinary.toString)) + getSecretValueResult.get.getSecretBinary.toString + } + val keyMap = parse(secretString).extract[Map[String, String]] + val creds = parseCredentials(keyMap) + Some(if (creds.contains("AWSCredential")) { + creds("AWSCredential") + } else { + new DefaultAWSCredential(Map("credentialName" -> name, "credentialValue" -> secretString)) }) } catch { @@ -60,6 +70,7 @@ class AWSCredentialParser extends CredentialParser { parameters.foldLeft(List[Credential]())((credentials, param) => { param._1 match { case "accessKeyId" => credentials :+ new AWSBasicCredential(parameters) + case "accountId" if !parameters.contains("accessKeyId") => credentials :+ new AWSBasicCredential(parameters) case "cloudWatchAccessKeyId" => credentials :+ new AWSCloudWatchCredential(parameters) case "dynamoDBAccessKeyId" => credentials :+ new AWSDynamoDBCredential(parameters) case _ => credentials diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3DataConnector.scala b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3DataConnector.scala new file mode 100644 index 00000000..ad78642f --- /dev/null +++ b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3DataConnector.scala @@ -0,0 +1,43 @@ +package com.acxiom.aws.pipeline.connectors + +import com.acxiom.aws.utils.{AWSCredential, S3Utilities} +import com.acxiom.pipeline.connectors.{BatchDataConnector, DataConnectorUtilities} +import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} +import com.acxiom.pipeline.{Credential, PipelineContext} +import org.apache.spark.sql.DataFrame + +case class S3DataConnector(override val name: String, + override val credentialName: Option[String], + override val credential: Option[Credential], + override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), + override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) extends BatchDataConnector { + override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { + val path = source.getOrElse("") + setSecurity(pipelineContext, path) + + DataConnectorUtilities.buildDataFrameReader(pipelineContext.sparkSession.get, readOptions) + .load(S3Utilities.replaceProtocol(path, S3Utilities.deriveProtocol(path))) + } + + override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Unit = { + val path = destination.getOrElse("") + setSecurity(pipelineContext, path) + + DataConnectorUtilities.buildDataFrameWriter(dataFrame, writeOptions) + .save(S3Utilities.replaceProtocol(path, S3Utilities.deriveProtocol(path))) + } + + private def setSecurity(pipelineContext: PipelineContext, path: String): Unit = { + val finalCredential = (if (credentialName.isDefined) { + pipelineContext.credentialProvider.get.getNamedCredential(credentialName.get) + } else { + credential + }).asInstanceOf[Option[AWSCredential]] + + if (finalCredential.isDefined) { + S3Utilities.setS3Authorization(path, + finalCredential.get.awsAccessKey, finalCredential.get.awsAccessSecret, + finalCredential.get.awsAccountId, finalCredential.get.awsRole, finalCredential.get.awsPartition, pipelineContext) + } + } +} diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/steps/S3Steps.scala b/metalus-aws/src/main/scala/com/acxiom/aws/steps/S3Steps.scala index c72561b1..55783065 100644 --- a/metalus-aws/src/main/scala/com/acxiom/aws/steps/S3Steps.scala +++ b/metalus-aws/src/main/scala/com/acxiom/aws/steps/S3Steps.scala @@ -4,7 +4,8 @@ import com.acxiom.aws.fs.S3FileManager import com.acxiom.aws.utils.S3Utilities import com.acxiom.pipeline.PipelineContext import com.acxiom.pipeline.annotations._ -import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameSteps, DataFrameWriterOptions} +import com.acxiom.pipeline.connectors.DataConnectorUtilities +import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} import com.amazonaws.services.s3.AmazonS3 import org.apache.spark.sql.DataFrame @@ -72,7 +73,7 @@ object S3Steps { options: Option[DataFrameReaderOptions] = None, pipelineContext: PipelineContext): DataFrame = { S3Utilities.setS3Authorization(path, accessKeyId, secretAccessKey, accountId, role, partition, pipelineContext) - DataFrameSteps.getDataFrameReader(options.getOrElse(DataFrameReaderOptions()), pipelineContext) + DataConnectorUtilities.buildDataFrameReader(pipelineContext.sparkSession.get, options.getOrElse(DataFrameReaderOptions())) .load(S3Utilities.replaceProtocol(path, S3Utilities.deriveProtocol(path))) } @@ -95,7 +96,7 @@ object S3Steps { options: Option[DataFrameReaderOptions] = None, pipelineContext: PipelineContext): DataFrame = { S3Utilities.setS3Authorization(paths.head, accessKeyId, secretAccessKey, accountId, role, partition, pipelineContext) - DataFrameSteps.getDataFrameReader(options.getOrElse(DataFrameReaderOptions()), pipelineContext) + DataConnectorUtilities.buildDataFrameReader(pipelineContext.sparkSession.get, options.getOrElse(DataFrameReaderOptions())) .load(paths.map(p => S3Utilities.replaceProtocol(p, S3Utilities.deriveProtocol(p))): _*) } @@ -119,7 +120,7 @@ object S3Steps { options: Option[DataFrameWriterOptions] = None, pipelineContext: PipelineContext): Unit = { S3Utilities.setS3Authorization(path, accessKeyId, secretAccessKey, accountId, role, partition, pipelineContext) - DataFrameSteps.getDataFrameWriter(dataFrame, options.getOrElse(DataFrameWriterOptions())) + DataConnectorUtilities.buildDataFrameWriter(dataFrame, options.getOrElse(DataFrameWriterOptions())) .save(S3Utilities.replaceProtocol(path, S3Utilities.deriveProtocol(path))) } diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/utils/AWSCredential.scala b/metalus-aws/src/main/scala/com/acxiom/aws/utils/AWSCredential.scala index 6cc6f00e..83a7f5cd 100644 --- a/metalus-aws/src/main/scala/com/acxiom/aws/utils/AWSCredential.scala +++ b/metalus-aws/src/main/scala/com/acxiom/aws/utils/AWSCredential.scala @@ -51,9 +51,47 @@ trait AWSCredential extends Credential { class DefaultAWSCredential(override val parameters: Map[String, Any]) extends AWSCredential { private implicit val formats: Formats = DefaultFormats override def name: String = parameters("credentialName").asInstanceOf[String] - private val keyMap = parse(parameters("credentialValue").asInstanceOf[String]).extract[Map[String, String]] - override def awsAccessKey: Option[String] = Some(keyMap.head._1) - override def awsAccessSecret: Option[String] = Some(keyMap.head._2) + private val keyMap = parse(parameters.getOrElse("credentialValue", "{}").asInstanceOf[String]).extract[Map[String, String]] + override def awsRole: Option[String] = if (keyMap.contains("role")) { + keyMap.get("role") + } else if (parameters.contains("role")) { + parameters.get("role").asInstanceOf[Option[String]] + } else { None } + override def awsAccountId: Option[String] = if (keyMap.contains("accountId")) { + keyMap.get("accountId") + } else if (parameters.contains("accountId")) { + parameters.get("accountId").asInstanceOf[Option[String]] + } else { None } + // See if the key is stored by name, then see if this is a role key and then use the default + override def awsAccessKey: Option[String] = if (keyMap.contains("accessKeyId")) { + keyMap.get("accessKeyId") + } else if ( parameters.contains("accessKeyId")) { + parameters.get("accessKeyId").asInstanceOf[Option[String]] + } else if (awsRole.isDefined) { + None + } else { Some(keyMap.head._1) } + override def awsAccessSecret: Option[String] = if (keyMap.contains("secretAccessKey")) { + keyMap.get("secretAccessKey") + } else if (parameters.contains("secretAccessKey")) { + parameters.get("secretAccessKey").asInstanceOf[Option[String]] + } else if (awsRole.isDefined) { + None + } else { Some(keyMap.head._2) } + override def sessionName: Option[String] = if (keyMap.contains("session")) { + keyMap.get("session") + } else if (parameters.contains("session")) { + parameters.get("session").asInstanceOf[Option[String]] + } else { None } + override def awsPartition: Option[String] = if (keyMap.contains("partition")) { + keyMap.get("partition") + } else if (parameters.contains("partition")) { + parameters.get("partition").asInstanceOf[Option[String]] + } else { None } + override def externalId: Option[String] = if (keyMap.contains("externalId")) { + keyMap.get("externalId") + } else if (parameters.contains("externalId")) { + parameters.get("externalId").asInstanceOf[Option[String]] + } else { None } } class AWSBasicCredential(override val parameters: Map[String, Any]) extends AWSCredential { diff --git a/metalus-common/docs/dataconnectorsteps.md b/metalus-common/docs/dataconnectorsteps.md new file mode 100644 index 00000000..cdacb542 --- /dev/null +++ b/metalus-common/docs/dataconnectorsteps.md @@ -0,0 +1,19 @@ +[Documentation Home](../../docs/readme.md) | [Common Home](../readme.md) + +# DataConnectorSteps +This step object provides a way to load from and write using DataConnectors. There are several step functions provided: + +## Write DataFrame +This function will write a given DataFrame using the provided DataConnector. Full parameter descriptions are listed below: + +* **dataFrame** - A dataFrame to be written. +* **connector** - The DataConnector to use when writing data. +* **destination** - An optional destination. +* **options** - Optional DataFrameWriterOptions object to configure the DataFrameWriter + +## Load DataFrame +This function will load a DataFrame using the provided DataConnector. Full parameter descriptions are listed below: + +* **source** - An optional source. +* **connector** - The DataConnector to use when loading data. +* **options** - Optional DataFrameReaderOptions object to configure the DataFrameReader diff --git a/metalus-common/readme.md b/metalus-common/readme.md index 25dfb2b2..4324f7cd 100644 --- a/metalus-common/readme.md +++ b/metalus-common/readme.md @@ -7,6 +7,7 @@ using Spark. ## Step Classes * [ApiSteps](docs/apisteps.md) * [CSVSteps](docs/csvsteps.md) +* [DataConnectorSteps](docs/dataconnectorsteps.md) * [DataFrameSteps](docs/dataframesteps.md) * [DataSteps](docs/datasteps.md) * [FileManagerSteps](docs/filemanagersteps.md) diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala new file mode 100644 index 00000000..d634c82d --- /dev/null +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala @@ -0,0 +1,20 @@ +package com.acxiom.pipeline.connectors + +import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} +import com.acxiom.pipeline.{Credential, PipelineContext} +import org.apache.spark.sql.DataFrame + +trait DataConnector { + def name: String + def credentialName: Option[String] + def credential: Option[Credential] + def load(source: Option[String], pipelineContext: PipelineContext): DataFrame + def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Unit +} + +trait BatchDataConnector extends DataConnector { + def readOptions: DataFrameReaderOptions = DataFrameReaderOptions() + def writeOptions: DataFrameWriterOptions = DataFrameWriterOptions() +} + +trait StreamingDataConnector extends DataConnector {} diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnectorUtilities.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnectorUtilities.scala new file mode 100644 index 00000000..d937d213 --- /dev/null +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnectorUtilities.scala @@ -0,0 +1,54 @@ +package com.acxiom.pipeline.connectors + +import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} +import org.apache.spark.sql.{DataFrameReader, DataFrameWriter, Dataset, SparkSession} + +object DataConnectorUtilities { + /** + * + * @param sparkSession The current spark session to use. + * @param options A DataFrameReaderOptions object for configuring the reader. + * @return A DataFrameReader based on the provided options. + */ + def buildDataFrameReader(sparkSession: SparkSession, options: DataFrameReaderOptions): DataFrameReader = { + val reader = sparkSession.read + .format(options.format) + .options(options.options.getOrElse(Map[String, String]())) + + if (options.schema.isDefined) { + reader.schema(options.schema.get.toStructType()) + } else { + reader + } + } + + /** + * + * @param dataFrame A DataFrame to write. + * @param options A DataFrameWriterOptions object for configuring the writer. + * @return A DataFrameWriter[Row] based on the provided options. + */ + def buildDataFrameWriter[T](dataFrame: Dataset[T], options: DataFrameWriterOptions): DataFrameWriter[T] = { + val writer = dataFrame.write.format(options.format) + .mode(options.saveMode) + .options(options.options.getOrElse(Map[String, String]())) + + val w1 = if (options.bucketingOptions.isDefined && options.bucketingOptions.get.columns.nonEmpty) { + val bucketingOptions = options.bucketingOptions.get + writer.bucketBy(bucketingOptions.numBuckets, bucketingOptions.columns.head, bucketingOptions.columns.drop(1): _*) + } else { + writer + } + val w2 = if (options.partitionBy.isDefined && options.partitionBy.get.nonEmpty) { + w1.partitionBy(options.partitionBy.get: _*) + } else { + w1 + } + if (options.sortBy.isDefined && options.sortBy.get.nonEmpty) { + val sortBy = options.sortBy.get + w2.sortBy(sortBy.head, sortBy.drop(1): _*) + } else { + w2 + } + } +} diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSDataConnector.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSDataConnector.scala new file mode 100644 index 00000000..4c3541b3 --- /dev/null +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSDataConnector.scala @@ -0,0 +1,18 @@ +package com.acxiom.pipeline.connectors + +import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} +import com.acxiom.pipeline.{Credential, PipelineContext} +import org.apache.spark.sql.DataFrame + +case class HDFSDataConnector(override val name: String, + override val credentialName: Option[String], + override val credential: Option[Credential], + override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), + override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) extends BatchDataConnector { + + override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = + DataConnectorUtilities.buildDataFrameReader(pipelineContext.sparkSession.get, readOptions).load(source.getOrElse("")) + + override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Unit = + DataConnectorUtilities.buildDataFrameWriter(dataFrame, writeOptions).save(destination.getOrElse("")) +} diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataConnectorSteps.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataConnectorSteps.scala new file mode 100644 index 00000000..2d36b408 --- /dev/null +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataConnectorSteps.scala @@ -0,0 +1,37 @@ +package com.acxiom.pipeline.steps + +import com.acxiom.pipeline.PipelineContext +import com.acxiom.pipeline.annotations.{StepFunction, StepObject, StepParameter, StepParameters} +import com.acxiom.pipeline.connectors.DataConnector +import org.apache.spark.sql.DataFrame + +@StepObject +object DataConnectorSteps { + + @StepFunction("836aab38-1140-4606-ab73-5b6744f0e7e7", + "Load", + "This step will create a DataFrame using the given DataConnector", + "Pipeline", + "Connectors") + @StepParameters(Map("dataFrame" -> StepParameter(None, Some(true), None, None, None, None, Some("The DataFrame to write")), + "connector" -> StepParameter(None, Some(true), None, None, None, None, Some("The data connector to use when writing")), + "source" -> StepParameter(None, Some(false), None, None, None, None, Some("The source path to load data")))) + def loadDataFrame(connector: DataConnector, + source: Option[String], + pipelineContext: PipelineContext): DataFrame = + connector.load(source, pipelineContext) + + @StepFunction("5608eba7-e9ff-48e6-af77-b5e810b99d89", + "Write", + "This step will write a DataFrame using the given DataConnector", + "Pipeline", + "Connectors") + @StepParameters(Map("dataFrame" -> StepParameter(None, Some(true), None, None, None, None, Some("The DataFrame to write")), + "connector" -> StepParameter(None, Some(true), None, None, None, None, Some("The data connector to use when writing")), + "destination" -> StepParameter(None, Some(false), None, None, None, None, Some("The destination path to write data")))) + def writeDataFrame(dataFrame: DataFrame, + connector: DataConnector, + destination: Option[String], + pipelineContext: PipelineContext): Unit = + connector.write(dataFrame, destination, pipelineContext) +} diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataFrameSteps.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataFrameSteps.scala index b9be2901..0211cd2f 100644 --- a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataFrameSteps.scala +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataFrameSteps.scala @@ -2,6 +2,7 @@ package com.acxiom.pipeline.steps import com.acxiom.pipeline.PipelineContext import com.acxiom.pipeline.annotations._ +import com.acxiom.pipeline.connectors.DataConnectorUtilities import org.apache.spark.sql._ import org.apache.spark.sql.functions.expr import org.apache.spark.storage.StorageLevel @@ -20,7 +21,7 @@ object DataFrameSteps { secondaryTypes = None) def getDataFrameReader(dataFrameReaderOptions: DataFrameReaderOptions, pipelineContext: PipelineContext): DataFrameReader = { - buildDataFrameReader(pipelineContext.sparkSession.get, dataFrameReaderOptions) + DataConnectorUtilities.buildDataFrameReader(pipelineContext.sparkSession.get, dataFrameReaderOptions) } @StepFunction("66a451c8-ffbd-4481-9c37-71777c3a240f", @@ -61,7 +62,7 @@ object DataFrameSteps { secondaryTypes = None) def getDataFrameWriter[T](dataFrame: Dataset[T], options: DataFrameWriterOptions): DataFrameWriter[T] = { - buildDataFrameWriter(dataFrame, options) + DataConnectorUtilities.buildDataFrameWriter(dataFrame, options) } @StepFunction("9aa6ae9f-cbeb-4b36-ba6a-02eee0a46558", @@ -173,54 +174,6 @@ object DataFrameSteps { dataFrame.repartition(partitions) } } - - /** - * - * @param sparkSession The current spark session to use. - * @param options A DataFrameReaderOptions object for configuring the reader. - * @return A DataFrameReader based on the provided options. - */ - private def buildDataFrameReader(sparkSession: SparkSession, options: DataFrameReaderOptions): DataFrameReader = { - val reader = sparkSession.read - .format(options.format) - .options(options.options.getOrElse(Map[String, String]())) - - if (options.schema.isDefined) { - reader.schema(options.schema.get.toStructType()) - } else { - reader - } - } - - /** - * - * @param dataFrame A DataFrame to write. - * @param options A DataFrameWriterOptions object for configuring the writer. - * @return A DataFrameWriter[Row] based on the provided options. - */ - private def buildDataFrameWriter[T](dataFrame: Dataset[T], options: DataFrameWriterOptions): DataFrameWriter[T] = { - val writer = dataFrame.write.format(options.format) - .mode(options.saveMode) - .options(options.options.getOrElse(Map[String, String]())) - - val w1 = if (options.bucketingOptions.isDefined && options.bucketingOptions.get.columns.nonEmpty) { - val bucketingOptions = options.bucketingOptions.get - writer.bucketBy(bucketingOptions.numBuckets, bucketingOptions.columns.head, bucketingOptions.columns.drop(1): _*) - } else { - writer - } - val w2 = if (options.partitionBy.isDefined && options.partitionBy.get.nonEmpty) { - w1.partitionBy(options.partitionBy.get: _*) - } else { - w1 - } - if (options.sortBy.isDefined && options.sortBy.get.nonEmpty) { - val sortBy = options.sortBy.get - w2.sortBy(sortBy.head, sortBy.drop(1): _*) - } else { - w2 - } - } } /** diff --git a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/DataConnectorStepsTests.scala b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/DataConnectorStepsTests.scala new file mode 100644 index 00000000..c84955ff --- /dev/null +++ b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/DataConnectorStepsTests.scala @@ -0,0 +1,214 @@ +package com.acxiom.pipeline.steps + +import com.acxiom.pipeline._ +import com.acxiom.pipeline.connectors.HDFSDataConnector +import org.apache.commons.io.FileUtils +import org.apache.hadoop.fs.FileSystem +import org.apache.hadoop.hdfs.{HdfsConfiguration, MiniDFSCluster} +import org.apache.log4j.{Level, Logger} +import org.apache.spark.SparkConf +import org.apache.spark.sql.SparkSession +import org.scalatest.{BeforeAndAfterAll, FunSpec} + +import java.io.{File, PrintWriter} +import java.nio.file.{Files, Path} +import scala.io.Source + +class DataConnectorStepsTests extends FunSpec with BeforeAndAfterAll { + val MASTER = "local[2]" + val APPNAME = "hdfs-steps-spark" + var sparkConf: SparkConf = _ + var sparkSession: SparkSession = _ + var pipelineContext: PipelineContext = _ + val sparkLocalDir: Path = Files.createTempDirectory("sparkLocal") + var config: HdfsConfiguration = _ + var fs: FileSystem = _ + var miniCluster: MiniDFSCluster = _ + val file = new File(sparkLocalDir.toFile.getAbsolutePath, "cluster") + + override def beforeAll(): Unit = { + Logger.getLogger("org.apache.spark").setLevel(Level.WARN) + Logger.getLogger("org.apache.hadoop").setLevel(Level.WARN) + Logger.getLogger("com.acxiom.pipeline").setLevel(Level.DEBUG) + + // set up mini hadoop cluster + config = new HdfsConfiguration() + config.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, file.getAbsolutePath) + miniCluster = new MiniDFSCluster.Builder(config).build() + miniCluster.waitActive() + // Only pull the fs object from the mini cluster + fs = miniCluster.getFileSystem + + sparkConf = new SparkConf() + .setMaster(MASTER) + .setAppName(APPNAME) + .set("spark.local.dir", sparkLocalDir.toFile.getAbsolutePath) + // Force Spark to use the HDFS cluster + .set("spark.hadoop.fs.defaultFS", miniCluster.getFileSystem().getUri.toString) + // Create the session + sparkSession = SparkSession.builder().config(sparkConf).getOrCreate() + + pipelineContext = PipelineContext(Some(sparkConf), Some(sparkSession), Some(Map[String, Any]()), + PipelineSecurityManager(), + PipelineParameters(List(PipelineParameter("0", Map[String, Any]()), PipelineParameter("1", Map[String, Any]()))), + Some(List("com.acxiom.pipeline.steps")), + PipelineStepMapper(), + Some(DefaultPipelineListener()), + Some(sparkSession.sparkContext.collectionAccumulator[PipelineStepMessage]("stepMessages"))) + } + + override def afterAll(): Unit = { + sparkSession.sparkContext.cancelAllJobs() + sparkSession.sparkContext.stop() + sparkSession.stop() + miniCluster.shutdown() + + Logger.getRootLogger.setLevel(Level.INFO) + // cleanup spark directories + FileUtils.deleteDirectory(sparkLocalDir.toFile) + } + + describe("HDFS Steps - Basic Writing") { + val chickens = Seq( + ("1", "silkie"), + ("2", "polish"), + ("3", "sultan") + ) + + it("should successfully write to hdfs") { + val spark = this.sparkSession + import spark.implicits._ + + val dataFrame = chickens.toDF("id", "chicken") + + val connector = HDFSDataConnector("my-connector", None, None, + DataFrameReaderOptions(), + DataFrameWriterOptions(format = "csv")) + DataConnectorSteps.writeDataFrame(dataFrame, connector, Some(miniCluster.getURI + "/data/chickens.csv"), pipelineContext) + val list = readHDFSContent(fs, miniCluster.getURI + "/data/chickens.csv") + + assert(list.size == 3) + + var writtenData: Seq[(String, String)] = Seq() + list.foreach(l => { + val tuple = l.split(',') + writtenData = writtenData ++ Seq((tuple(0), tuple(1))) + }) + + writtenData.sortBy(t => t._1) + + assert(writtenData == chickens) + } + it("should respect options") { + val spark = this.sparkSession + import spark.implicits._ + + val dataFrame = chickens.toDF("id", "chicken") + DataFrameSteps.persistDataFrame(dataFrame, "MEMORY_ONLY") + val connector = HDFSDataConnector("my-connector", None, None, + DataFrameReaderOptions(), + DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) + DataConnectorSteps.writeDataFrame(dataFrame, connector, Some(miniCluster.getURI + "/data/chickens.csv"), pipelineContext) + DataFrameSteps.unpersistDataFrame(dataFrame) + val list = readHDFSContent(fs, miniCluster.getURI + "/data/chickens.csv") + + assert(list.size == 3) + + var writtenData: Seq[(String, String)] = Seq() + list.foreach(l => { + val tuple = l.split('þ') + writtenData = writtenData ++ Seq((tuple(0), tuple(1))) + }) + + writtenData.sortBy(t => t._1) + + assert(writtenData == chickens) + } + + it("Should save a DataFrame") { + val spark = this.sparkSession + import spark.implicits._ + + val dataFrame = chickens.toDF("id", "chicken") + val path = miniCluster.getURI + "/data/chickens.csv" + val connector = HDFSDataConnector("my-connector", None, None, + DataFrameReaderOptions(), + DataFrameWriterOptions(format = "csv", saveMode = "overwrite")) + DataConnectorSteps.writeDataFrame(dataFrame, connector, Some(path), pipelineContext) + val list = readHDFSContent(fs, miniCluster.getURI + "/data/chickens.csv") + assert(list.size == 3) + var writtenData: Seq[(String, String)] = Seq() + list.foreach(l => { + val tuple = l.split(',') + writtenData = writtenData ++ Seq((tuple(0), tuple(1))) + }) + writtenData.sortBy(t => t._1) + assert(writtenData == chickens) + } + } + + describe("HDFS Steps - Basic Reading") { + val chickens = Seq( + ("1", "silkie"), + ("2", "polish"), + ("3", "sultan") + ) + + it("should successfully read from hdfs") { + val csv = "1,silkie\n2,polish\n3,sultan" + val path = miniCluster.getURI + "/data/chickens2.csv" + + writeHDFSContext(fs, path, csv) + val connector = HDFSDataConnector("my-connector", None, None, + DataFrameReaderOptions(format = "csv")) + val dataFrame = DataConnectorSteps.loadDataFrame(connector, Some(path), pipelineContext) + DataFrameSteps.persistDataFrame(dataFrame) + assert(dataFrame.count() == 3) + val result = dataFrame.take(3).map(r => (r.getString(0), r.getString(1))).toSeq + DataFrameSteps.unpersistDataFrame(dataFrame, blocking = true) + assert(result == chickens) + } + + it("should respect options") { + val csv = "idþchicken\n1þsilkie\n2þpolish\n3þsultan" + val path = miniCluster.getURI + "/data/chickens2.csv" + + writeHDFSContext(fs, path, csv) + val connector = HDFSDataConnector("my-connector", None, None, + DataFrameReaderOptions(format = "csv", options = Some(Map("header" -> "true", "delimiter" -> "þ"))), + DataFrameWriterOptions()) + val dataFrame = DataConnectorSteps.loadDataFrame(connector, Some(path), pipelineContext) + + assert(dataFrame.count() == 3) + val result = dataFrame.take(3).map(r => (r.getString(0), r.getString(1))).toSeq + assert(result == chickens) + } + + it("Should load a a DataFrame") { + val csv = "1,silkie\n2,polish\n3,sultan" + val path = miniCluster.getURI + "/data/chickens2.csv" + writeHDFSContext(fs, path, csv) + val connector = HDFSDataConnector("my-connector", None, None, + DataFrameReaderOptions(format = "csv"), + DataFrameWriterOptions()) + val dataFrame = DataConnectorSteps.loadDataFrame(connector, Some(path), pipelineContext) + val results = dataFrame.collect().map(r => (r.getString(0), r.getString(1))).toSeq + assert(results.size == 3) + assert(results == chickens) + } + } + + private def readHDFSContent(fs: FileSystem, path: String): List[String] = { + assert(fs.exists(new org.apache.hadoop.fs.Path(path))) + + val statuses = fs.globStatus(new org.apache.hadoop.fs.Path(path + "/part*")) + statuses.foldLeft(List[String]())((list, stat) => list ::: Source.fromInputStream(fs.open(stat.getPath)).getLines.toList) + } + + private def writeHDFSContext(fs: FileSystem, path: String, content: String): Unit = { + + val pw = new PrintWriter(fs.create(new org.apache.hadoop.fs.Path(path))) + pw.print(content) + pw.close() + } +} diff --git a/metalus-core/src/main/scala/com/acxiom/pipeline/CredentialProvider.scala b/metalus-core/src/main/scala/com/acxiom/pipeline/CredentialProvider.scala index b1a0727a..27297a29 100644 --- a/metalus-core/src/main/scala/com/acxiom/pipeline/CredentialProvider.scala +++ b/metalus-core/src/main/scala/com/acxiom/pipeline/CredentialProvider.scala @@ -71,7 +71,13 @@ class DefaultCredentialProvider(override val parameters: Map[String, Any]) exten defaultParsers } } - protected lazy val credentials: Map[String, Credential] = { + protected lazy val credentials: Map[String, Credential] = parseCredentials(parameters) + + override def getNamedCredential(name: String): Option[Credential] = { + this.credentials.get(name) + } + + protected def parseCredentials(parameters: Map[String, Any]): Map[String, Credential] = { credentialParsers.foldLeft(Map[String, Credential]())((credentials, parser) => { val creds = parser.parseCredentials(parameters) if (creds.nonEmpty) { @@ -83,10 +89,6 @@ class DefaultCredentialProvider(override val parameters: Map[String, Any]) exten } }) } - - override def getNamedCredential(name: String): Option[Credential] = { - this.credentials.get(name) - } } /** diff --git a/metalus-gcp/docs/gcpsecretsmanager-credentialprovider.md b/metalus-gcp/docs/gcpsecretsmanager-credentialprovider.md index 6bcad6fc..857f3528 100644 --- a/metalus-gcp/docs/gcpsecretsmanager-credentialprovider.md +++ b/metalus-gcp/docs/gcpsecretsmanager-credentialprovider.md @@ -4,3 +4,9 @@ This [CredentialProvider](../../docs/credentialprovider.md) implementation extends the [DefaultCredentialProvider](../../docs/credentialprovider.md#DefaultCredentialProvider) by searching the GCP Secrets Manager for the named secret. A [BasicCredential](../../docs/credentialprovider.md#BasicCredential) will be returned containing the string value. A _projectId_ is required to instantiate. + +## Secrets Manager Formats +When creating the secret that will be used, there are several properties that will be considered. Since GCP Secrets Manager +takes a single value, when storing the JSON service account key, it should be stored as JSON. The same is applicable when +storing [AWS keys](../../metalus-aws/docs/awssecretsmanager-credentialprovider.md#secrets-manager-formats). The data should +not be encoded. diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPCredential.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPCredential.scala index f75e0d0f..5bbbaec3 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPCredential.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPCredential.scala @@ -11,6 +11,14 @@ trait GCPCredential extends Credential { def authKey: Map[String, String] } +/** + * An implementation of GCPCredential where parameters represent the full authKey. + * @param parameters The actual JSON authKey + */ +class BasicGCPCredential(override val parameters: Map[String, Any]) extends GCPCredential { + override def authKey: Map[String, String] = parameters +} + /** * GCPCredential implementation that looks for the gcpAuthKeyArray parameter to generate the authKey. * @param parameters A map containing the gcpAuthKeyArray parameter diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPSecretsManagerCredentialProvider.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPSecretsManagerCredentialProvider.scala index 5c39234b..bb8d5637 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPSecretsManagerCredentialProvider.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPSecretsManagerCredentialProvider.scala @@ -3,6 +3,8 @@ package com.acxiom.gcp.pipeline import com.acxiom.pipeline._ import com.google.cloud.secretmanager.v1.{SecretManagerServiceClient, SecretName, SecretVersion, SecretVersionName} import org.apache.log4j.Logger +import org.json4s.native.JsonMethods.parse +import org.json4s.{DefaultFormats, Formats} import java.util.Base64 import scala.collection.JavaConverters._ @@ -15,11 +17,11 @@ import scala.collection.JavaConverters._ class GCPSecretsManagerCredentialProvider(override val parameters: Map[String, Any]) extends DefaultCredentialProvider(parameters) { private val logger = Logger.getLogger(getClass) + private implicit val formats: Formats = DefaultFormats override protected val defaultParsers = List(new DefaultCredentialParser(), new GCPCredentialParser) private val projectId = parameters.getOrElse("projectId", "").asInstanceOf[String] private val secretsManagerClient = SecretManagerServiceClient.create() - override def getNamedCredential(name: String): Option[Credential] = { val baseCredential = this.credentials.get(name) @@ -40,7 +42,14 @@ class GCPSecretsManagerCredentialProvider(override val parameters: Map[String, A val secret = Option(secretsManagerClient.accessSecretVersion(SecretVersionName.of(projectId, name, recent.toString))) if (secret.isDefined && Option(secret.get.getPayload.getData.toStringUtf8).isDefined) { - Some(DefaultCredential(Map("credentialName" -> name, "credentialValue" -> secret.get.getPayload.getData.toStringUtf8))) + val secretString = secret.get.getPayload.getData.toStringUtf8 + val credsMap = if ("(\\{.*?})".r.findAllIn(secretString).hasNext) { + parse(secretString).extract[Map[String, String]] + } else { + Map("credentialName" -> name, "credentialValue" -> secretString) + } + val creds = parseCredentials(credsMap) + Some(creds.head._2) } else { None } @@ -77,6 +86,8 @@ class GCPCredentialParser() extends CredentialParser { } else { credentialList :+ DefaultCredential(parameters) } + } else if (parameters.contains("project_id") && parameters.contains("auth_uri")) { + credentialList :+ new BasicGCPCredential(parameters) } else { credentialList } diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala new file mode 100644 index 00000000..5c1a51fb --- /dev/null +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala @@ -0,0 +1,40 @@ +package com.acxiom.gcp.pipeline.connectors + +import com.acxiom.gcp.fs.GCSFileManager +import com.acxiom.gcp.pipeline.GCPCredential +import com.acxiom.gcp.utils.GCPUtilities +import com.acxiom.pipeline.connectors.{BatchDataConnector, DataConnectorUtilities} +import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} +import com.acxiom.pipeline.{Credential, PipelineContext} +import org.apache.spark.sql.DataFrame + +case class GCSDataConnector(override val name: String, + override val credentialName: Option[String], + override val credential: Option[Credential], + override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), + override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) extends BatchDataConnector { + override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { + val path = source.getOrElse("") + setSecurity(pipelineContext) + DataConnectorUtilities.buildDataFrameReader(pipelineContext.sparkSession.get, readOptions) + .load(GCSFileManager.prepareGCSFilePath(path)) + } + + override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Unit = { + val path = destination.getOrElse("") + setSecurity(pipelineContext) + DataConnectorUtilities.buildDataFrameWriter(dataFrame, writeOptions) + .save(GCSFileManager.prepareGCSFilePath(path)) + } + + private def setSecurity(pipelineContext: PipelineContext): Unit = { + val finalCredential = (if (credentialName.isDefined) { + pipelineContext.credentialProvider.get.getNamedCredential(credentialName.get) + } else { + credential + }).asInstanceOf[Option[GCPCredential]] + if (finalCredential.isDefined) { + GCPUtilities.setGCSAuthorization(finalCredential.get.authKey, pipelineContext) + } + } +} diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/GCSSteps.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/GCSSteps.scala index d1c111c0..f7a7c58f 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/GCSSteps.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/GCSSteps.scala @@ -1,9 +1,11 @@ package com.acxiom.gcp.steps import com.acxiom.gcp.fs.GCSFileManager +import com.acxiom.gcp.utils.GCPUtilities import com.acxiom.pipeline.PipelineContext import com.acxiom.pipeline.annotations._ -import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameSteps, DataFrameWriterOptions} +import com.acxiom.pipeline.connectors.DataConnectorUtilities +import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} import org.apache.spark.sql.DataFrame import org.json4s.native.Serialization import org.json4s.{DefaultFormats, Formats} @@ -43,10 +45,9 @@ object GCSSteps { options: Option[DataFrameReaderOptions] = None, pipelineContext: PipelineContext): DataFrame = { if (credentials.isDefined) { - setGCSAuthorization(credentials.get, pipelineContext) + GCPUtilities.setGCSAuthorization(credentials.get, pipelineContext) } - DataFrameSteps - .getDataFrameReader(options.getOrElse(DataFrameReaderOptions()), pipelineContext) + DataConnectorUtilities.buildDataFrameReader(pipelineContext.sparkSession.get, options.getOrElse(DataFrameReaderOptions())) .load(paths.map(GCSFileManager.prepareGCSFilePath(_)): _*) } @@ -65,9 +66,9 @@ object GCSSteps { options: Option[DataFrameWriterOptions] = None, pipelineContext: PipelineContext): Unit = { if (credentials.isDefined) { - setGCSAuthorization(credentials.get, pipelineContext) + GCPUtilities.setGCSAuthorization(credentials.get, pipelineContext) } - DataFrameSteps.getDataFrameWriter(dataFrame, options.getOrElse(DataFrameWriterOptions())) + DataConnectorUtilities.buildDataFrameWriter(dataFrame, options.getOrElse(DataFrameWriterOptions())) .save(GCSFileManager.prepareGCSFilePath(path)) } @@ -92,22 +93,4 @@ object GCSSteps { secondaryTypes = None) def createFileManager(projectId: String, bucket: String, credentials: Map[String, String]): Option[GCSFileManager] = Some(new GCSFileManager(projectId, bucket, Some(Serialization.write(credentials)))) - - /** - * Given a credential map, this function will set the appropriate properties required for Spark access. - * - * @param credentials The GCP auth map - * @param pipelineContext The current pipeline context - */ - private def setGCSAuthorization(credentials: Map[String, String], pipelineContext: PipelineContext): Unit = { - val sc = pipelineContext.sparkSession.get.sparkContext - sc.hadoopConfiguration.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem") - sc.hadoopConfiguration.set("fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS") - // Private Key - sc.hadoopConfiguration.set("fs.gs.project.id", credentials("project_id")) - sc.hadoopConfiguration.set("fs.gs.auth.service.account.enable", "true") - sc.hadoopConfiguration.set("fs.gs.auth.service.account.email", credentials("client_email")) - sc.hadoopConfiguration.set("fs.gs.auth.service.account.private.key.id", credentials("private_key_id")) - sc.hadoopConfiguration.set("fs.gs.auth.service.account.private.key", credentials("private_key")) - } } diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/utils/GCPUtilities.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/utils/GCPUtilities.scala index 00cb2b71..6c8d9128 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/utils/GCPUtilities.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/utils/GCPUtilities.scala @@ -26,6 +26,24 @@ object GCPUtilities { .setMaxRpcTimeout(Duration.ofMinutes(Constants.ONE)) .setTotalTimeout(Duration.ofMinutes(Constants.TWO)).build + /** + * Given a credential map, this function will set the appropriate properties required for Spark access. + * + * @param credentials The GCP auth map + * @param pipelineContext The current pipeline context + */ + def setGCSAuthorization(credentials: Map[String, String], pipelineContext: PipelineContext): Unit = { + val sc = pipelineContext.sparkSession.get.sparkContext + sc.hadoopConfiguration.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem") + sc.hadoopConfiguration.set("fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS") + // Private Key + sc.hadoopConfiguration.set("fs.gs.project.id", credentials("project_id")) + sc.hadoopConfiguration.set("fs.gs.auth.service.account.enable", "true") + sc.hadoopConfiguration.set("fs.gs.auth.service.account.email", credentials("client_email")) + sc.hadoopConfiguration.set("fs.gs.auth.service.account.private.key.id", credentials("private_key_id")) + sc.hadoopConfiguration.set("fs.gs.auth.service.account.private.key", credentials("private_key")) + } + /** * Retrieve the credentials needed to interact with GCP services. * @param credentialProvider The credential provider From 490420cb524a865a6368b6c187a986200ea83928 Mon Sep 17 00:00:00 2001 From: dafreels Date: Tue, 7 Sep 2021 10:02:35 -0400 Subject: [PATCH 04/24] #252 Fixed compilation issue with new credential --- .../src/main/scala/com/acxiom/gcp/pipeline/GCPCredential.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPCredential.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPCredential.scala index 5bbbaec3..bd0f5102 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPCredential.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/GCPCredential.scala @@ -16,7 +16,7 @@ trait GCPCredential extends Credential { * @param parameters The actual JSON authKey */ class BasicGCPCredential(override val parameters: Map[String, Any]) extends GCPCredential { - override def authKey: Map[String, String] = parameters + override def authKey: Map[String, String] = parameters.map(record => record._1 -> record._2.toString) } /** From 9c78f2595df65cbea6385c4454b4eb0e62787be4 Mon Sep 17 00:00:00 2001 From: dafreels Date: Thu, 9 Sep 2021 08:31:30 -0400 Subject: [PATCH 05/24] #256 Added ability to specify the credential name to use for connecting to the streaming source. --- metalus-aws/docs/kinesispipelinedriver.md | 7 ++++++ .../aws/drivers/KinesisPipelineDriver.scala | 25 +++++++++++-------- metalus-gcp/docs/pubsubpipelinedriver.md | 5 ++++ .../gcp/drivers/PubSubPipelineDriver.scala | 2 +- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/metalus-aws/docs/kinesispipelinedriver.md b/metalus-aws/docs/kinesispipelinedriver.md index 42ad2517..f050acd6 100644 --- a/metalus-aws/docs/kinesispipelinedriver.md +++ b/metalus-aws/docs/kinesispipelinedriver.md @@ -8,6 +8,10 @@ is consumed, the RDD will be converted into a DataFrame with three columns: * **value** - the data * **topic** - The appName +The driver will attempt to locate a credential using the optional credential name parameters below. If the parameters +aren't specified, then the defaults are used. Finally the client will be created without using credentials and rely +on the credentials used to start the Spark job. + ## Command line Parameters *Required parameters:* * **driverSetupClass** - This class will handle all of the initial setup such as building out pipelines, creating the PipelineContext. @@ -16,6 +20,9 @@ is consumed, the RDD will be converted into a DataFrame with three columns: *Optional Parameters:* * **appName** - The optional name of this app to use when check pointing Kinesis sequence numbers. +* **kinesisCredentialName** - An optional name of the credential to use when building the Kinesis client. Default AWSCredential +* **kinesisCloudWatchCredentialName** - An optional name of the credential to use for Cloud Watch when building the Kinesis client. Default AWSCloudWatchCredential +* **kinesisDynamoDBCredentialName** - An optional name of the credential to use for DynamoDB when building the Kinesis client. Default AWSDynamoDBCredential * **accessKeyId** - The AWS access key used to connect * **secretAccessKey** - The AWS access secret used to connect * **consumerStreams** - [number] The number of streams to create. Defaults to the number of shards defined for the stream. diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/drivers/KinesisPipelineDriver.scala b/metalus-aws/src/main/scala/com/acxiom/aws/drivers/KinesisPipelineDriver.scala index e4a99d94..6f50ce43 100644 --- a/metalus-aws/src/main/scala/com/acxiom/aws/drivers/KinesisPipelineDriver.scala +++ b/metalus-aws/src/main/scala/com/acxiom/aws/drivers/KinesisPipelineDriver.scala @@ -52,11 +52,11 @@ object KinesisPipelineDriver { val appName = parameters.getOrElse("appName", s"Metalus_Kinesis_$streamName(${new Date().getTime})").asInstanceOf[String] // Get the credential provider val credentialProvider = driverSetup.credentialProvider - val awsCredential = credentialProvider.getNamedCredential("AWSCredential").asInstanceOf[Option[AWSCredential]] + val awsCredential = credentialProvider.getNamedCredential( + parameters.getOrElse("kinesisCredentialName", "AWSCredential").toString).asInstanceOf[Option[AWSCredential]] val duration = StreamingUtils.getDuration(parameters.get("duration-type").map(_.asInstanceOf[String]), parameters.get("duration").map(_.asInstanceOf[String])) - val streamingContext = - StreamingUtils.createStreamingContext(sparkSession.sparkContext, Some(duration)) + val streamingContext = StreamingUtils.createStreamingContext(sparkSession.sparkContext, Some(duration)) // Get the client val kinesisClient = KinesisUtilities.buildKinesisClient(region, awsCredential) // Handle multiple shards @@ -64,7 +64,8 @@ object KinesisPipelineDriver { logger.info("Number of Kinesis shards is : " + numShards) val numStreams = parameters.getOrElse("consumerStreams", numShards).toString.toInt // Create the Kinesis DStreams - val kinesisStreams = createKinesisDStreams(credentialProvider, appName, duration, streamingContext, numStreams, region, streamName) + val kinesisStreams = createKinesisDStreams(awsCredential, credentialProvider, appName, duration, + streamingContext, numStreams, region, streamName, parameters) logger.info("Created " + kinesisStreams.size + " Kinesis DStreams") val defaultParser = new KinesisStreamingDataParser(appName) val streamingParsers = StreamingUtils.generateStreamingDataParsers(parameters, Some(List(defaultParser))) @@ -73,8 +74,7 @@ object KinesisPipelineDriver { allStreams.foreachRDD { (rdd: RDD[Record], time: Time) => logger.debug(s"Checking RDD for data(${time.toString()}): ${rdd.count()}") if (streamingParameters.processEmptyRDD || !rdd.isEmpty()) { - logger.debug("RDD received") - // Convert the RDD into a dataFrame + logger.debug("RDD received") // Convert the RDD into a dataFrame val parser = StreamingUtils.getStreamingParser[Record](rdd, streamingParsers) val dataFrame = parser.getOrElse(defaultParser).parseRDD(rdd, sparkSession) // Refresh the execution plan prior to processing new data @@ -87,11 +87,14 @@ object KinesisPipelineDriver { logger.info("Shutting down Kinesis Pipeline Driver") } - private def createKinesisDStreams(credentialProvider: CredentialProvider, appName: String, duration: Duration, - streamingContext: StreamingContext, numStreams: Int, region: String, streamName: String) = { - val awsCredential = credentialProvider.getNamedCredential("AWSCredential").asInstanceOf[Option[AWSCredential]] - val cloudWatchCredential = credentialProvider.getNamedCredential("AWSCloudWatchCredential").asInstanceOf[Option[AWSCredential]] - val dynamoDBCredential = credentialProvider.getNamedCredential("AWSDynamoDBCredential").asInstanceOf[Option[AWSCredential]] + private def createKinesisDStreams(awsCredential: Option[AWSCredential], credentialProvider: CredentialProvider, appName: String, duration: Duration, + streamingContext: StreamingContext, numStreams: Int, region: String, streamName: String, + parameters: Map[String, Any]) = { + val cloudWatchCredential = credentialProvider.getNamedCredential( + parameters.getOrElse("kinesisCloudWatchCredentialName", "AWSCloudWatchCredential").toString + ).asInstanceOf[Option[AWSCredential]] + val dynamoDBCredential = credentialProvider.getNamedCredential( + parameters.getOrElse("kinesisDynamoDBCredentialName", "AWSDynamoDBCredential").toString).asInstanceOf[Option[AWSCredential]] (0 until numStreams).map { _ => val builder = KinesisInputDStream.builder .endpointUrl(s"https://kinesis.$region.amazonaws.com") diff --git a/metalus-gcp/docs/pubsubpipelinedriver.md b/metalus-gcp/docs/pubsubpipelinedriver.md index 092b12c1..f806d60d 100644 --- a/metalus-gcp/docs/pubsubpipelinedriver.md +++ b/metalus-gcp/docs/pubsubpipelinedriver.md @@ -8,6 +8,10 @@ streams. As data is consumed, the RDD will be converted into a DataFrame with th * **value** - the data * **topic** - The appName +The driver will attempt to locate a credential using the optional credential name parameter below. If the parameter +isn't specified, then the default is used. Finally the client will be created without using credentials and rely +on the credentials used to start the Spark job. + ## Command line Parameters *Required parameters:* * **driverSetupClass** - This class will handle all of the initial setup such as building out pipelines, creating the PipelineContext. @@ -15,6 +19,7 @@ streams. As data is consumed, the RDD will be converted into a DataFrame with th * **subscription** - The Pub/Sub subscription to listen for messages. *Optional Parameters:* +* **pubsubCredentialName** - An optional name of the credential to use when building the PubSub client. Default GCPCredential * **duration-type** - [minutes | **seconds**] Corresponds to the *duration* parameter. * **duration** - [number] How long the driver should wait before processing the next batch of data. Default is 10 seconds. * **terminationPeriod** - [number] The number of ms the system should run and then shut down. diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/drivers/PubSubPipelineDriver.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/drivers/PubSubPipelineDriver.scala index 26fbff71..fd95eee5 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/drivers/PubSubPipelineDriver.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/drivers/PubSubPipelineDriver.scala @@ -33,7 +33,7 @@ object PubSubPipelineDriver { Some(parameters.getOrElse("duration", "10").asInstanceOf[String])) // Get the credential provider val credentialProvider = driverSetup.credentialProvider - val gcpCredential = credentialProvider.getNamedCredential("GCPCredential") + val gcpCredential = credentialProvider.getNamedCredential(parameters.getOrElse("pubsubCredentialName", "GCPCredential").toString) val sparkGCPCredentials = if (gcpCredential.isDefined) { SparkGCPCredentials.builder.jsonServiceAccount( GCPUtilities.generateCredentialsByteArray(Some(gcpCredential.get.asInstanceOf[GCPCredential].authKey)).get).build() From 60044b06c4199de088404d73b8c1f1d2e64b40a3 Mon Sep 17 00:00:00 2001 From: dafreels Date: Mon, 13 Sep 2021 10:23:30 -0400 Subject: [PATCH 06/24] #252 Implemented streaming to batch and batch to streaming within the connectors. Implemented the Kinesis connector. --- docs/dataconnectors.md | 43 +++++++- .../connectors/AWSDataConnector.scala | 15 +++ .../connectors/KinesisDataConnector.scala | 90 +++++++++++++++++ .../pipeline/connectors/KinesisWriter.scala | 97 +++++++++++++++++++ .../pipeline/connectors/S3DataConnector.scala | 28 +++--- .../com/acxiom/aws/steps/KinesisSteps.scala | 39 ++------ .../acxiom/aws/utils/KinesisUtilities.scala | 48 ++++++++- .../pipeline/connectors/DataConnector.scala | 10 +- .../connectors/HDFSDataConnector.scala | 15 ++- .../pipeline/steps/DataConnectorSteps.scala | 3 +- .../connectors/GCSDataConnector.scala | 16 ++- 11 files changed, 348 insertions(+), 56 deletions(-) create mode 100644 metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/AWSDataConnector.scala create mode 100644 metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisDataConnector.scala create mode 100644 metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisWriter.scala diff --git a/docs/dataconnectors.md b/docs/dataconnectors.md index 9aa0e298..ea87acdc 100644 --- a/docs/dataconnectors.md +++ b/docs/dataconnectors.md @@ -7,6 +7,15 @@ to load and write a DataFrame based on the underlying system. Below is a breakdo ![DataConnectors](images/DataConnectors.png) +**Parameters** +The following parameters are available to all data connectors: + +* **name** - The name of the connector +* **credentialName** - The optional credential name to use to authenticate +* **credential** - The optional credential to use to authenticate +* **readOptions** - The optional read options to use when loading the DataFrame +* **writeOptions** - The optional write options to use when writing the DataFrame + ## Batch Connectors that are designed to load and write data for batch processing will extend the _BatchDataConnector_. These are very straightforward and offer the most reusable components. @@ -99,7 +108,39 @@ val connector = GCSDataConnector("my-connector", Some("my-credential-name-for-se } } ``` -## Streaming (Coming Soon) +## Streaming Streaming connectors offer a way to use pipelines with [Spark Structured Streaming](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html) without the need to write new [drivers](pipeline-drivers.md). When designing pipelines for streaming, care must be taken to not inject steps that are more batch oriented such as doing a file copy. + +### KinesisDataConnector +This connector provides access to Kinesis. In addition to the standard parameters, the following parameters are +available: + +* **streamName** - The name of the Kinesis stream. +* **region** - The region containing the Kinesis stream +* **partitionKey** - The optional static partition key to use +* **partitionKeyIndex** - The optional field index in the DataFrame row containing the value to use as the partition key +* **separator** - The field separator to use when formatting the row data + +Below is an example setup that expects a secrets manager credential provider: +#### Scala +```scala +val connector = KinesisDataConnector("stream-name", "us-east-1", None, Some(15), ",", "my-connector", + Some("my-credential-name-for-secrets-manager")) +``` +#### Globals JSON +```json +{ + "connector": { + "className": "com.acxiom.aws.pipeline.connectors.S3DataConnector", + "object": { + "name": "my-connector", + "credentialName": "my-credential-name-for-secrets-manager", + "streamName": "stream-name", + "region": "us-east-1", + "separator": "," + } + } +} +``` diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/AWSDataConnector.scala b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/AWSDataConnector.scala new file mode 100644 index 00000000..53663fd1 --- /dev/null +++ b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/AWSDataConnector.scala @@ -0,0 +1,15 @@ +package com.acxiom.aws.pipeline.connectors + +import com.acxiom.aws.utils.AWSCredential +import com.acxiom.pipeline.PipelineContext +import com.acxiom.pipeline.connectors.DataConnector + +trait AWSDataConnector extends DataConnector{ + protected def getCredential(pipelineContext: PipelineContext): Option[AWSCredential] = { + (if (credentialName.isDefined) { + pipelineContext.credentialProvider.get.getNamedCredential(credentialName.get) + } else { + credential + }).asInstanceOf[Option[AWSCredential]] + } +} diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisDataConnector.scala b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisDataConnector.scala new file mode 100644 index 00000000..7f4ecd43 --- /dev/null +++ b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisDataConnector.scala @@ -0,0 +1,90 @@ +package com.acxiom.aws.pipeline.connectors + +import com.acxiom.aws.utils.{AWSCredential, KinesisUtilities} +import com.acxiom.pipeline.connectors.StreamingDataConnector +import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} +import com.acxiom.pipeline.{Credential, PipelineContext} +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.streaming.StreamingQuery + +/** + * Data Connector implementation that works with Kinesis. Each row produced will be formatted to a string using + * the separator character provided. + * + * @param streamName The name of the Kinesis stream. + * @param region The region containing the Kinesis stream + * @param partitionKey The optional static partition key to use + * @param partitionKeyIndex The optional field index in the DataFrame row containing the value to use as the partition key + * @param separator The field separator to use when formatting the row data + * @param name The name of the connector + * @param credentialName The optional name of the credential to use when authorizing to the Kinesis stream + * @param credential The optional credential to use when authorizing to the Kinesis stream + * @param readOptions The optional read options to use + * @param writeOptions The optional write options to use + */ +case class KinesisDataConnector(streamName: String, + region: String = "us-east-1", + partitionKey: Option[String], + partitionKeyIndex: Option[Int], + separator: String = ",", + override val name: String, + override val credentialName: Option[String], + override val credential: Option[Credential], + override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), + override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) + extends StreamingDataConnector with AWSDataConnector { + override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { + val initialReader = pipelineContext.sparkSession.get.readStream + .format("kinesis") + .option("streamName", streamName) + .option("region", region) + .option("initialPosition", "TRIM_HORIZON") + .options(readOptions.options.getOrElse(Map[String, String]())) + + val reader = if (readOptions.schema.isDefined) { + initialReader.schema(readOptions.schema.get.toStructType()) + } else { + initialReader + } + + val finalCredential: Option[AWSCredential] = getCredential(pipelineContext) + + val finalReader = if (finalCredential.isDefined) { + if (finalCredential.get.awsRole.isDefined && finalCredential.get.awsAccountId.isDefined) { + val r1 = if (finalCredential.get.externalId.isDefined) { + reader.option("roleExternalId", finalCredential.get.externalId.get) + } else { + reader + } + (if (finalCredential.get.sessionName.isDefined) { + r1.option("roleSessionName", finalCredential.get.sessionName.get) + } else { + r1 + }).option("roleArn", finalCredential.get.awsRoleARN.get) + } else { + reader + .option("awsAccessKey", finalCredential.get.awsAccessKey.getOrElse("")) + .option("awsSecretKey", finalCredential.get.awsAccessSecret.getOrElse("")) + } + reader + } else { + reader + } + finalReader.load() + } + + override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { + val finalCredential: Option[AWSCredential] = getCredential(pipelineContext) + if (dataFrame.isStreaming) { + Some(dataFrame + .writeStream + .format(writeOptions.format) + .options(writeOptions.options.getOrElse(Map[String, String]())) + .foreach(new StructuredStreamingKinesisSink(streamName, region, partitionKey, partitionKeyIndex, separator, finalCredential)) + .start()) + } else { + KinesisUtilities.writeDataFrame(dataFrame, region, streamName, partitionKey, partitionKeyIndex, separator, finalCredential) + None + } + } +} diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisWriter.scala b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisWriter.scala new file mode 100644 index 00000000..739421a9 --- /dev/null +++ b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisWriter.scala @@ -0,0 +1,97 @@ +package com.acxiom.aws.pipeline.connectors + +import com.acxiom.aws.utils.{AWSCredential, KinesisUtilities} +import com.amazonaws.services.kinesis.AmazonKinesis +import com.amazonaws.services.kinesis.model.{PutRecordsRequest, PutRecordsRequestEntry} +import org.apache.spark.sql.{ForeachWriter, Row} + +import java.nio.ByteBuffer +import scala.collection.mutable.ArrayBuffer + +/** + * Write a batch DataFrame to Kinesis using record batching. The following parameters are expected: + * streamName The Kinesis stream name + * region The region of the Kinesis stream + * dataFrame The DataFrame to write + * partitionKey The static partition key to use + * partitionKeyIndex The field index in the DataFrame row containing the value to use as the partition key + * separator The field separator to use when formatting the row data + * credential An optional credential to use to authenticate to Kinesis + */ +trait KinesisWriter { + // Kinesis Client Limits + val maxBufferSize: Int = 500 * 1024 + val maxRecords = 500 + // Buffers + protected val buffer = new ArrayBuffer[PutRecordsRequestEntry]() + protected var bufferSize: Long = 0L + // Client library + protected var kinesisClient: AmazonKinesis = _ + + def streamName: String + def region: String + def credential: Option[AWSCredential] + def partitionKey: Option[String] + def partitionKeyIndex: Option[Int] + def separator: String + + def open(): Unit = { + kinesisClient = KinesisUtilities.buildKinesisClient(region, credential) + } + + def close(): Unit = { + if (buffer.nonEmpty) { + flush() + } + kinesisClient.shutdown() + } + + def process(value: Row): Unit = { + val data = value.mkString(separator).getBytes + if ((data.length + bufferSize > maxBufferSize && buffer.nonEmpty) || buffer.length == maxRecords) { + flush() + } + val putRecordRequest = new PutRecordsRequestEntry() + buffer += (if (partitionKey.isDefined) { + putRecordRequest.withPartitionKey(partitionKey.get) + } else if (partitionKeyIndex.isDefined) { + putRecordRequest.withPartitionKey(value.getAs[Any](partitionKeyIndex.get).toString) + } else { + putRecordRequest + }).withData(ByteBuffer.wrap(data)) + bufferSize += data.length + } + + private def flush(): Unit = { + val recordRequest = new PutRecordsRequest() + .withStreamName(streamName) + .withRecords(buffer: _*) + + kinesisClient.putRecords(recordRequest) + buffer.clear() + bufferSize = 0 + } +} + +class BatchKinesisWriter(override val streamName: String, + override val region: String, + override val partitionKey: Option[String], + override val partitionKeyIndex: Option[Int], + override val separator: String, + override val credential: Option[AWSCredential]) extends KinesisWriter + +class StructuredStreamingKinesisSink(override val streamName: String, + override val region: String, + override val partitionKey: Option[String], + override val partitionKeyIndex: Option[Int], + override val separator: String, + override val credential: Option[AWSCredential]) extends ForeachWriter[Row] with KinesisWriter { + override def open(partitionId: Long, epochId: Long): Boolean = { + this.open() + true + } + + override def process(value: Row): Unit = super.process(_) + + override def close(errorOrNull: Throwable): Unit = this.close() +} diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3DataConnector.scala b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3DataConnector.scala index ad78642f..f90c25cf 100644 --- a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3DataConnector.scala +++ b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3DataConnector.scala @@ -1,16 +1,18 @@ package com.acxiom.aws.pipeline.connectors -import com.acxiom.aws.utils.{AWSCredential, S3Utilities} +import com.acxiom.aws.utils.S3Utilities import com.acxiom.pipeline.connectors.{BatchDataConnector, DataConnectorUtilities} import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} import com.acxiom.pipeline.{Credential, PipelineContext} import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.streaming.StreamingQuery case class S3DataConnector(override val name: String, override val credentialName: Option[String], override val credential: Option[Credential], override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), - override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) extends BatchDataConnector { + override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) + extends BatchDataConnector with AWSDataConnector { override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { val path = source.getOrElse("") setSecurity(pipelineContext, path) @@ -19,20 +21,24 @@ case class S3DataConnector(override val name: String, .load(S3Utilities.replaceProtocol(path, S3Utilities.deriveProtocol(path))) } - override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Unit = { + override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { val path = destination.getOrElse("") setSecurity(pipelineContext, path) - - DataConnectorUtilities.buildDataFrameWriter(dataFrame, writeOptions) - .save(S3Utilities.replaceProtocol(path, S3Utilities.deriveProtocol(path))) + if (dataFrame.isStreaming) { + Some(dataFrame.writeStream + .format(writeOptions.format) + .option("path", destination.getOrElse("")) + .options(writeOptions.options.getOrElse(Map[String, String]())) + .start()) + } else { + DataConnectorUtilities.buildDataFrameWriter(dataFrame, writeOptions) + .save(S3Utilities.replaceProtocol(path, S3Utilities.deriveProtocol(path))) + None + } } private def setSecurity(pipelineContext: PipelineContext, path: String): Unit = { - val finalCredential = (if (credentialName.isDefined) { - pipelineContext.credentialProvider.get.getNamedCredential(credentialName.get) - } else { - credential - }).asInstanceOf[Option[AWSCredential]] + val finalCredential = getCredential(pipelineContext) if (finalCredential.isDefined) { S3Utilities.setS3Authorization(path, diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/steps/KinesisSteps.scala b/metalus-aws/src/main/scala/com/acxiom/aws/steps/KinesisSteps.scala index 63c3e842..542e3f35 100644 --- a/metalus-aws/src/main/scala/com/acxiom/aws/steps/KinesisSteps.scala +++ b/metalus-aws/src/main/scala/com/acxiom/aws/steps/KinesisSteps.scala @@ -1,6 +1,6 @@ package com.acxiom.aws.steps -import com.acxiom.aws.utils.{AWSUtilities, KinesisUtilities} +import com.acxiom.aws.utils.{AWSBasicCredential, AWSUtilities, KinesisUtilities} import com.acxiom.pipeline.PipelineContext import com.acxiom.pipeline.annotations.{StepFunction, StepObject, StepParameter, StepParameters} import org.apache.spark.sql.DataFrame @@ -26,13 +26,11 @@ object KinesisSteps { separator: String = ",", accessKeyId: Option[String] = None, secretAccessKey: Option[String] = None): Unit = { - val index = determinePartitionKey(dataFrame, partitionKey) - dataFrame.rdd.foreach(row => { - val rowData = row.mkString(separator) - val key = row.getAs[Any](index).toString - postMessage(rowData, region, streamName, key, accessKeyId, secretAccessKey) - }) + val index = KinesisUtilities.determinePartitionKey(dataFrame, partitionKey) + val creds = Some(new AWSBasicCredential(Map("accessKeyId" -> accessKeyId, "secretAccessKey" -> secretAccessKey))) + KinesisUtilities.writeDataFrame(dataFrame, region, streamName, None, Some(index), separator, creds) } + @StepFunction("5c9c7056-5c7a-4463-93c8-7e99bad66d4f", "Write DataFrame to a Kinesis Stream Using Global Credentials", "This step will write a DataFrame to a Kinesis Stream using the CredentialProvider to get Credentials", @@ -49,13 +47,9 @@ object KinesisSteps { partitionKey: String, separator: String = ",", pipelineContext: PipelineContext): Unit = { - val index = determinePartitionKey(dataFrame, partitionKey) + val index = KinesisUtilities.determinePartitionKey(dataFrame, partitionKey) val creds = AWSUtilities.getAWSCredential(pipelineContext.credentialProvider) - dataFrame.rdd.foreach(row => { - val rowData = row.mkString(separator) - val key = row.getAs[Any](index).toString - KinesisUtilities.postMessageWithCredentials(rowData, region, streamName, key, creds) - }) + KinesisUtilities.writeDataFrame(dataFrame, region, streamName, None, Some(index), separator, creds) } @StepFunction("52f161a5-3025-4e40-a10b-f201940b5cbf", @@ -95,23 +89,4 @@ object KinesisSteps { secretAccessKey: Option[String] = None): Unit = { KinesisUtilities.postMessage(message, region, streamName, partitionKey, accessKeyId, secretAccessKey) } - - /** - * Determines the column id to use to extract the partition key value when writing rows - * @param dataFrame The DataFrame containing the schema - * @param partitionKey The field name of the column to use for the key value. - * @return The column index or zero id the column name is not found. - */ - private def determinePartitionKey(dataFrame: DataFrame, partitionKey: String): Int = { - if (dataFrame.schema.isEmpty) { - 0 - } else { - val field = dataFrame.schema.fieldIndex(partitionKey) - if (field < 0) { - 0 - } else { - field - } - } - } } diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/utils/KinesisUtilities.scala b/metalus-aws/src/main/scala/com/acxiom/aws/utils/KinesisUtilities.scala index dfa92948..557c8296 100644 --- a/metalus-aws/src/main/scala/com/acxiom/aws/utils/KinesisUtilities.scala +++ b/metalus-aws/src/main/scala/com/acxiom/aws/utils/KinesisUtilities.scala @@ -1,10 +1,12 @@ package com.acxiom.aws.utils +import com.acxiom.aws.pipeline.connectors.BatchKinesisWriter import com.amazonaws.auth.{AWSCredentials, AWSStaticCredentialsProvider, BasicAWSCredentials} import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration -import com.amazonaws.regions.Regions import com.amazonaws.services.kinesis.model.PutRecordRequest import com.amazonaws.services.kinesis.{AmazonKinesis, AmazonKinesisClient} +import org.apache.spark.sql.DataFrame + import java.nio.ByteBuffer object KinesisUtilities { @@ -42,6 +44,25 @@ object KinesisUtilities { kinesisClient.build() } + /** + * Determines the column id to use to extract the partition key value when writing rows + * @param dataFrame The DataFrame containing the schema + * @param partitionKey The field name of the column to use for the key value. + * @return The column index or zero id the column name is not found. + */ + def determinePartitionKey(dataFrame: DataFrame, partitionKey: String): Int = { + if (dataFrame.schema.isEmpty) { + 0 + } else { + val field = dataFrame.schema.fieldIndex(partitionKey) + if (field < 0) { + 0 + } else { + field + } + } + } + /** * Write a single message to a Kinesis Stream * @param message The message to post to the Kinesis stream @@ -87,4 +108,29 @@ object KinesisUtilities { kinesisClient.putRecord(putRecordRequest) kinesisClient.shutdown() } + + /** + * Write a batch DataFrame to Kinesis using record batching. + * @param dataFrame The DataFrame to write + * @param region The region of the Kinesis stream + * @param streamName The Kinesis stream name + * @param partitionKey The static partition key to use + * @param partitionKeyIndex The field index in the DataFrame row containing the value to use as the partition key + * @param separator The field separator to use when formatting the row data + * @param credential An optional credential to use to authenticate to Kinesis + */ + def writeDataFrame(dataFrame: DataFrame, + region: String, + streamName: String, + partitionKey: Option[String], + partitionKeyIndex: Option[Int], + separator: String = ",", + credential: Option[AWSCredential] = None): Unit = { + dataFrame.rdd.foreachPartition(rows => { + val writer = new BatchKinesisWriter(streamName, region, partitionKey, partitionKeyIndex, separator, credential) + writer.open() + rows.foreach(writer.process) + writer.close() + }) + } } diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala index d634c82d..ecf0b265 100644 --- a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala @@ -3,18 +3,18 @@ package com.acxiom.pipeline.connectors import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} import com.acxiom.pipeline.{Credential, PipelineContext} import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.streaming.StreamingQuery trait DataConnector { def name: String def credentialName: Option[String] def credential: Option[Credential] - def load(source: Option[String], pipelineContext: PipelineContext): DataFrame - def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Unit -} - -trait BatchDataConnector extends DataConnector { def readOptions: DataFrameReaderOptions = DataFrameReaderOptions() def writeOptions: DataFrameWriterOptions = DataFrameWriterOptions() + def load(source: Option[String], pipelineContext: PipelineContext): DataFrame + def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] } +trait BatchDataConnector extends DataConnector {} + trait StreamingDataConnector extends DataConnector {} diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSDataConnector.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSDataConnector.scala index 4c3541b3..8b2052fe 100644 --- a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSDataConnector.scala +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSDataConnector.scala @@ -3,6 +3,7 @@ package com.acxiom.pipeline.connectors import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} import com.acxiom.pipeline.{Credential, PipelineContext} import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.streaming.StreamingQuery case class HDFSDataConnector(override val name: String, override val credentialName: Option[String], @@ -13,6 +14,16 @@ case class HDFSDataConnector(override val name: String, override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = DataConnectorUtilities.buildDataFrameReader(pipelineContext.sparkSession.get, readOptions).load(source.getOrElse("")) - override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Unit = - DataConnectorUtilities.buildDataFrameWriter(dataFrame, writeOptions).save(destination.getOrElse("")) + override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { + if (dataFrame.isStreaming) { + Some(dataFrame.writeStream + .format(writeOptions.format) + .option("path", destination.getOrElse("")) + .options(writeOptions.options.getOrElse(Map[String, String]())) + .start()) + } else { + DataConnectorUtilities.buildDataFrameWriter(dataFrame, writeOptions).save(destination.getOrElse("")) + None + } + } } diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataConnectorSteps.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataConnectorSteps.scala index 2d36b408..8a4e6820 100644 --- a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataConnectorSteps.scala +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataConnectorSteps.scala @@ -4,6 +4,7 @@ import com.acxiom.pipeline.PipelineContext import com.acxiom.pipeline.annotations.{StepFunction, StepObject, StepParameter, StepParameters} import com.acxiom.pipeline.connectors.DataConnector import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.streaming.StreamingQuery @StepObject object DataConnectorSteps { @@ -32,6 +33,6 @@ object DataConnectorSteps { def writeDataFrame(dataFrame: DataFrame, connector: DataConnector, destination: Option[String], - pipelineContext: PipelineContext): Unit = + pipelineContext: PipelineContext): Option[StreamingQuery] = connector.write(dataFrame, destination, pipelineContext) } diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala index 5c1a51fb..6f2582d4 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala @@ -7,6 +7,7 @@ import com.acxiom.pipeline.connectors.{BatchDataConnector, DataConnectorUtilitie import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} import com.acxiom.pipeline.{Credential, PipelineContext} import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.streaming.StreamingQuery case class GCSDataConnector(override val name: String, override val credentialName: Option[String], @@ -20,11 +21,20 @@ case class GCSDataConnector(override val name: String, .load(GCSFileManager.prepareGCSFilePath(path)) } - override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Unit = { + override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { val path = destination.getOrElse("") setSecurity(pipelineContext) - DataConnectorUtilities.buildDataFrameWriter(dataFrame, writeOptions) - .save(GCSFileManager.prepareGCSFilePath(path)) + if (dataFrame.isStreaming) { + Some(dataFrame.writeStream + .format(writeOptions.format) + .option("path", destination.getOrElse("")) + .options(writeOptions.options.getOrElse(Map[String, String]())) + .start()) + } else { + DataConnectorUtilities.buildDataFrameWriter(dataFrame, writeOptions) + .save(GCSFileManager.prepareGCSFilePath(path)) + None + } } private def setSecurity(pipelineContext: PipelineContext): Unit = { From 7cc7f5cf2987093e26ddac8799ff3cd862f88fd7 Mon Sep 17 00:00:00 2001 From: dafreels Date: Mon, 13 Sep 2021 10:46:48 -0400 Subject: [PATCH 07/24] #252 Fixed an issue with the KinesisWriter --- .../com/acxiom/aws/pipeline/connectors/KinesisWriter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisWriter.scala b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisWriter.scala index 739421a9..86484dec 100644 --- a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisWriter.scala +++ b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisWriter.scala @@ -91,7 +91,7 @@ class StructuredStreamingKinesisSink(override val streamName: String, true } - override def process(value: Row): Unit = super.process(_) + override def process(value: Row): Unit = super.process(_: Row) override def close(errorOrNull: Throwable): Unit = this.close() } From b4c27dc26746d8daf36d21de80bcdc49cb69a6a5 Mon Sep 17 00:00:00 2001 From: dafreels Date: Tue, 14 Sep 2021 06:22:39 -0400 Subject: [PATCH 08/24] #252 Added BigQueryDataConnector --- docs/dataconnectors.md | 21 ++++++ .../connectors/AWSDataConnector.scala | 8 +-- .../pipeline/connectors/DataConnector.scala | 8 +++ .../connectors/BigQueryDataConnector.scala | 66 +++++++++++++++++ .../com/acxiom/gcp/steps/BigQuerySteps.scala | 70 +++++++------------ 5 files changed, 122 insertions(+), 51 deletions(-) create mode 100644 metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/BigQueryDataConnector.scala diff --git a/docs/dataconnectors.md b/docs/dataconnectors.md index ea87acdc..aefe44ba 100644 --- a/docs/dataconnectors.md +++ b/docs/dataconnectors.md @@ -108,6 +108,27 @@ val connector = GCSDataConnector("my-connector", Some("my-credential-name-for-se } } ``` +### BigQueryDataConnector +This connector provides access to BigQuery. Below is an example setup that expects a secrets manager credential provider: +#### Scala +```scala +val connector = BigDataConnector("temp-bucket-name", "my-connector", Some("my-credential-name-for-secrets-manager"), None, + DataFrameReaderOptions(format = "csv"), + DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) +``` +#### Globals JSON +```json +{ + "connector": { + "className": "com.acxiom.gcp.pipeline.connectors.GCSDataConnector", + "object": { + "name": "my-connector", + "credentialName": "my-credential-name-for-secrets-manager", + "tempWriteBucket": "temp-bucket-name" + } + } +} +``` ## Streaming Streaming connectors offer a way to use pipelines with [Spark Structured Streaming](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html) without the need to write new [drivers](pipeline-drivers.md). When designing pipelines for streaming, care must be taken to not diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/AWSDataConnector.scala b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/AWSDataConnector.scala index 53663fd1..57a8b3e4 100644 --- a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/AWSDataConnector.scala +++ b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/AWSDataConnector.scala @@ -5,11 +5,7 @@ import com.acxiom.pipeline.PipelineContext import com.acxiom.pipeline.connectors.DataConnector trait AWSDataConnector extends DataConnector{ - protected def getCredential(pipelineContext: PipelineContext): Option[AWSCredential] = { - (if (credentialName.isDefined) { - pipelineContext.credentialProvider.get.getNamedCredential(credentialName.get) - } else { - credential - }).asInstanceOf[Option[AWSCredential]] + override protected def getCredential(pipelineContext: PipelineContext): Option[AWSCredential] = { + super.getCredential(pipelineContext).asInstanceOf[Option[AWSCredential]] } } diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala index ecf0b265..b04eff41 100644 --- a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala @@ -13,6 +13,14 @@ trait DataConnector { def writeOptions: DataFrameWriterOptions = DataFrameWriterOptions() def load(source: Option[String], pipelineContext: PipelineContext): DataFrame def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] + + protected def getCredential(pipelineContext: PipelineContext): Option[Credential] = { + if (credentialName.isDefined) { + pipelineContext.credentialProvider.get.getNamedCredential(credentialName.get) + } else { + credential + } + } } trait BatchDataConnector extends DataConnector {} diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/BigQueryDataConnector.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/BigQueryDataConnector.scala new file mode 100644 index 00000000..88422d5a --- /dev/null +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/BigQueryDataConnector.scala @@ -0,0 +1,66 @@ +package com.acxiom.gcp.pipeline.connectors + +import com.acxiom.gcp.pipeline.GCPCredential +import com.acxiom.gcp.utils.GCPUtilities +import com.acxiom.pipeline.connectors.{BatchDataConnector, DataConnectorUtilities} +import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} +import com.acxiom.pipeline.{Credential, PipelineContext} +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.streaming.StreamingQuery + +import java.util.Base64 + +case class BigQueryDataConnector(tempWriteBucket: String, + override val name: String, + override val credentialName: Option[String], + override val credential: Option[Credential], + override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), + override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) extends BatchDataConnector { + override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { + val table = source.getOrElse("") + val readerOptions = readOptions.copy(format = "bigquery") + // Setup authentication + val finalCredential = getCredential(pipelineContext).asInstanceOf[Option[GCPCredential]] + val finalOptions = if (finalCredential.isDefined) { + readerOptions.copy(options = setBigQueryAuthentication(finalCredential.get, readerOptions.options)) + } else { + readerOptions + } + DataConnectorUtilities.buildDataFrameReader(pipelineContext.sparkSession.get, finalOptions).load(table) + } + + override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { + val table = destination.getOrElse("") + // Setup format for BigQuery + val writerOptions = writeOptions.copy(format = "bigquery", options = Some(Map("temporaryGcsBucket" -> tempWriteBucket))) + val finalCredential = getCredential(pipelineContext).asInstanceOf[Option[GCPCredential]] + val finalOptions = if (finalCredential.isDefined) { + writerOptions.copy(options = setBigQueryAuthentication(finalCredential.get, writerOptions.options)) + } else { + writerOptions + } + if (dataFrame.isStreaming) { + Some(dataFrame.writeStream.foreachBatch { (batchDF: DataFrame, batchId: Long) => + DataConnectorUtilities.buildDataFrameWriter(batchDF, finalOptions).save(table) + }.start()) + } else { + DataConnectorUtilities.buildDataFrameWriter(dataFrame, finalOptions).save(table) + None + } + } + + private def setBigQueryAuthentication(credentials: GCPCredential, + options: Option[Map[String, String]]): Option[Map[String, String]] = { + val creds = GCPUtilities.generateCredentialsByteArray(Some(credentials.authKey)) + if (creds.isDefined) { + val encodedCredential = Base64.getEncoder.encodeToString(creds.get) + if (options.isDefined) { + Some(options.get + ("credentials" -> encodedCredential)) + } else { + Some(Map("credentials" -> encodedCredential)) + } + } else { + options + } + } +} diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/BigQuerySteps.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/BigQuerySteps.scala index 1077da63..0d51922b 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/BigQuerySteps.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/BigQuerySteps.scala @@ -1,13 +1,12 @@ package com.acxiom.gcp.steps -import com.acxiom.gcp.utils.GCPUtilities +import com.acxiom.gcp.pipeline.BasicGCPCredential +import com.acxiom.gcp.pipeline.connectors.BigQueryDataConnector import com.acxiom.pipeline.PipelineContext import com.acxiom.pipeline.annotations._ -import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameSteps, DataFrameWriterOptions} +import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} import org.apache.spark.sql.DataFrame -import java.util.Base64 - @StepObject object BigQuerySteps { @StepFunction("3a91eee5-95c1-42dd-9c30-6edc0f9de1ca", @@ -20,22 +19,22 @@ object BigQuerySteps { "credentials" -> StepParameter(None, Some(false), None, None, None, None, Some("Optional credentials map")))) @StepResults(primaryType = "org.apache.spark.sql.DataFrame", secondaryTypes = None) def readFromTable(table: String, - credentials: Option[Map[String, String]], - options: Option[DataFrameReaderOptions] = None, - pipelineContext: PipelineContext): DataFrame = { + credentials: Option[Map[String, String]], + options: Option[DataFrameReaderOptions] = None, + pipelineContext: PipelineContext): DataFrame = { // Setup format for BigQuery val readerOptions = if (options.isDefined) { - options.get.copy(format = "bigquery") + options.get } else { - DataFrameReaderOptions("bigquery") + DataFrameReaderOptions() } - // Setup authentication - val finalOptions = if (credentials.isDefined) { - readerOptions.copy(options = setBigQueryAuthentication(credentials.get, readerOptions.options)) + val creds = if (credentials.isDefined) { + Some(new BasicGCPCredential(credentials.get)) } else { - readerOptions + None } - DataFrameSteps.getDataFrameReader(finalOptions, pipelineContext).load(table) + val connector = BigQueryDataConnector("", "readFromTable", None, creds, readerOptions) + connector.load(Some(table), pipelineContext) } @StepFunction("5b6e114b-51bb-406f-a95a-2a07bc0d05c7", @@ -49,42 +48,23 @@ object BigQuerySteps { "options" -> StepParameter(None, Some(false), None, None, None, None, Some("The optional DataFrame Options")), "credentials" -> StepParameter(None, Some(false), None, None, None, None, Some("Optional credentials map")))) def writeToTable(dataFrame: DataFrame, - table: String, - tempBucket: String, - credentials: Option[Map[String, String]], - options: Option[DataFrameWriterOptions] = None): Unit = { + table: String, + tempBucket: String, + credentials: Option[Map[String, String]], + options: Option[DataFrameWriterOptions] = None, + pipelineContext: PipelineContext): Unit = { // Setup format for BigQuery val writerOptions = if (options.isDefined) { - options.get.copy(format = "bigquery") + options.get } else { - DataFrameWriterOptions("bigquery", options = Some(Map("temporaryGcsBucket" -> tempBucket))) + DataFrameWriterOptions() } - // Setup authentication - val finalOptions = if (credentials.isDefined) { - val tempOptions = if (writerOptions.options.isDefined) { - writerOptions.options.get + ("temporaryGcsBucket" -> tempBucket) - } else { - Map("temporaryGcsBucket" -> tempBucket) - } - writerOptions.copy(options = setBigQueryAuthentication(credentials.get, Some(tempOptions))) - } else { - writerOptions - } - DataFrameSteps.getDataFrameWriter(dataFrame, finalOptions).save(table) - } - - private def setBigQueryAuthentication(credentials: Map[String, String], - options: Option[Map[String, String]]): Option[Map[String, String]] = { - val creds = GCPUtilities.generateCredentialsByteArray(Some(credentials)) - if (creds.isDefined) { - val encodedCredential = Base64.getEncoder.encodeToString(creds.get) - if (options.isDefined) { - Some(options.get + ("credentials" -> encodedCredential)) - } else { - Some(Map("credentials" -> encodedCredential)) - } + val creds = if (credentials.isDefined) { + Some(new BasicGCPCredential(credentials.get)) } else { - options + None } + val connector = BigQueryDataConnector(tempBucket, "writeToTable", None, creds, DataFrameReaderOptions(), writerOptions) + connector.write(dataFrame, Some(table), pipelineContext) } } From ed297eebfd558a425b6515cf4937c7cc5640194a Mon Sep 17 00:00:00 2001 From: dafreels Date: Tue, 14 Sep 2021 07:25:08 -0400 Subject: [PATCH 09/24] #252 Added KafkaDataConnector --- docs/dataconnectors.md | 31 +++++++++++ .../connectors/KafkaDataConnector.scala | 55 +++++++++++++++++++ .../com/acxiom/kafka/steps/KafkaSteps.scala | 35 +----------- .../acxiom/kafka/utils/KafkaUtilities.scala | 31 +++++++++++ 4 files changed, 120 insertions(+), 32 deletions(-) create mode 100644 metalus-kafka/src/main/scala/com/acxiom/kafka/pipeline/connectors/KafkaDataConnector.scala diff --git a/docs/dataconnectors.md b/docs/dataconnectors.md index aefe44ba..a274b912 100644 --- a/docs/dataconnectors.md +++ b/docs/dataconnectors.md @@ -165,3 +165,34 @@ val connector = KinesisDataConnector("stream-name", "us-east-1", None, Some(15), } } ``` +### KafkaDataConnector +This connector provides access to Kinesis. In addition to the standard parameters, the following parameters are +available: + +* **topics** - The name of the Kinesis stream. +* **kafkaNodes** - The region containing the Kinesis stream +* **key** - The optional static key to use +* **keyField** - The optional field name in the DataFrame row containing the value to use as the key +* **separator** - The field separator to use when formatting the row data +* +Below is an example setup that expects a secrets manager credential provider: +#### Scala +```scala +val connector = KafkaDataConnector("topic-name1,topic-name2", "host1:port1,host2:port2", "message-key", None, + "my-connector", Some("my-credential-name-for-secrets-manager")) +``` +#### Globals JSON +```json +{ + "connector": { + "className": "com.acxiom.aws.pipeline.connectors.S3DataConnector", + "object": { + "name": "my-connector", + "credentialName": "my-credential-name-for-secrets-manager", + "topics": "topic-name1,topic-name2", + "kafkaNodes": "host1:port1,host2:port2", + "key": "message-key" + } + } +} +``` diff --git a/metalus-kafka/src/main/scala/com/acxiom/kafka/pipeline/connectors/KafkaDataConnector.scala b/metalus-kafka/src/main/scala/com/acxiom/kafka/pipeline/connectors/KafkaDataConnector.scala new file mode 100644 index 00000000..48a2ff48 --- /dev/null +++ b/metalus-kafka/src/main/scala/com/acxiom/kafka/pipeline/connectors/KafkaDataConnector.scala @@ -0,0 +1,55 @@ +package com.acxiom.kafka.pipeline.connectors + +import com.acxiom.kafka.utils.KafkaUtilities +import com.acxiom.pipeline.connectors.StreamingDataConnector +import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} +import com.acxiom.pipeline.{Credential, PipelineContext} +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.functions.lit +import org.apache.spark.sql.streaming.StreamingQuery + +case class KafkaDataConnector(topics: String, + kafkaNodes: String, + key: Option[String], + keyField: Option[String], + override val name: String, + override val credentialName: Option[String], + override val credential: Option[Credential], + override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), + override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions(), + clientId: String = "metalus_default_kafka_producer_client", + separator: String = ",") extends StreamingDataConnector { + override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { + pipelineContext.sparkSession.get + .readStream + .format("kafka") + .option("kafka.bootstrap.servers", kafkaNodes) + .option("subscribe", topics) + .options(readOptions.options.getOrElse(Map[String, String]())) + .load() + } + + override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { + if (dataFrame.isStreaming) { + Some(dataFrame + .writeStream + .format("kafka") + .option("kafka.bootstrap.servers", kafkaNodes) + .option("subscribe", topics) + .options(writeOptions.options.getOrElse(Map[String, String]())) + .start()) + } else { + val col = if (keyField.isDefined && dataFrame.schema.fields.exists(_.name == keyField.get)) { + dataFrame.col(keyField.get) + } else if (key.isDefined) { + lit(key.get) + } else if (keyField.isDefined) { + lit(keyField.get) + } else { + lit("message_key") + } + KafkaUtilities.publishDataFrame(dataFrame, topics, kafkaNodes, col, separator, clientId) + None + } + } +} diff --git a/metalus-kafka/src/main/scala/com/acxiom/kafka/steps/KafkaSteps.scala b/metalus-kafka/src/main/scala/com/acxiom/kafka/steps/KafkaSteps.scala index 8169980a..e348bc5a 100644 --- a/metalus-kafka/src/main/scala/com/acxiom/kafka/steps/KafkaSteps.scala +++ b/metalus-kafka/src/main/scala/com/acxiom/kafka/steps/KafkaSteps.scala @@ -2,8 +2,8 @@ package com.acxiom.kafka.steps import com.acxiom.kafka.utils.KafkaUtilities import com.acxiom.pipeline.annotations.{StepFunction, StepObject, StepParameter, StepParameters} +import org.apache.spark.sql.DataFrame import org.apache.spark.sql.functions._ -import org.apache.spark.sql.{Column, DataFrame} @StepObject object KafkaSteps { @@ -29,7 +29,7 @@ object KafkaSteps { } else { lit(keyField) } - publishDataFrame(dataFrame, topic, kafkaNodes, col, separator, clientId) + KafkaUtilities.publishDataFrame(dataFrame, topic, kafkaNodes, col, separator, clientId) } @StepFunction("eaf68ea6-1c37-4427-85be-165ee9777c4d", @@ -49,7 +49,7 @@ object KafkaSteps { key: String, separator: String = ",", clientId: String = "metalus_default_kafka_producer_client"): Unit = { - publishDataFrame(dataFrame, topic, kafkaNodes, lit(key), separator, clientId) + KafkaUtilities.publishDataFrame(dataFrame, topic, kafkaNodes, lit(key), separator, clientId) } @StepFunction("74efe1e1-edd1-4c38-8e2b-bb693e3e3f4c", @@ -67,33 +67,4 @@ object KafkaSteps { kafkaNodes: String, key: String, clientId: String = "metalus_default_kafka_producer_client"): Unit = KafkaUtilities.postMessage(message, topic, kafkaNodes, key, clientId) - - /** - * Publish DataFrame data to a Kafka topic. - * - * @param dataFrame The DataFrame being published - * @param topic The Kafka topic - * @param kafkaNodes Comma separated list of kafka nodes - * @param key The Kafka key used for partitioning - * @param separator The field separator used to combine the columns. - * @param clientId The kafka clientId. - */ - private def publishDataFrame(dataFrame: DataFrame, - topic: String, - kafkaNodes: String, - key: Column, - separator: String = ",", - clientId: String = "metalus_default_kafka_producer_client"): Unit = { - val columns = dataFrame.schema.fields.foldLeft(List[Column]())((cols, field) => { - cols :+ dataFrame.col(field.name) :+ lit(separator) - }) - dataFrame.withColumn("topic", lit(topic)) - .withColumn("key", key) - .withColumn("value", concat(columns.dropRight(1): _*)) - .write - .format("kafka") - .option("kafka.bootstrap.servers", kafkaNodes) - .option("kafka.client.id", clientId) - .save() - } } diff --git a/metalus-kafka/src/main/scala/com/acxiom/kafka/utils/KafkaUtilities.scala b/metalus-kafka/src/main/scala/com/acxiom/kafka/utils/KafkaUtilities.scala index f5bc659f..390e4ee5 100644 --- a/metalus-kafka/src/main/scala/com/acxiom/kafka/utils/KafkaUtilities.scala +++ b/metalus-kafka/src/main/scala/com/acxiom/kafka/utils/KafkaUtilities.scala @@ -1,6 +1,8 @@ package com.acxiom.kafka.utils import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} +import org.apache.spark.sql.functions.{concat, lit} +import org.apache.spark.sql.{Column, DataFrame} import java.util.Properties @@ -37,4 +39,33 @@ object KafkaUtilities { producer.flush() producer.close() } + + /** + * Publish DataFrame data to a Kafka topic. + * + * @param dataFrame The DataFrame being published + * @param topic The Kafka topic + * @param kafkaNodes Comma separated list of kafka nodes + * @param key The Kafka key used for partitioning + * @param separator The field separator used to combine the columns. + * @param clientId The kafka clientId. + */ + def publishDataFrame(dataFrame: DataFrame, + topic: String, + kafkaNodes: String, + key: Column, + separator: String = ",", + clientId: String = "metalus_default_kafka_producer_client"): Unit = { + val columns = dataFrame.schema.fields.foldLeft(List[Column]())((cols, field) => { + cols :+ dataFrame.col(field.name) :+ lit(separator) + }) + dataFrame.withColumn("topic", lit(topic)) + .withColumn("key", key) + .withColumn("value", concat(columns.dropRight(1): _*)) + .write + .format("kafka") + .option("kafka.bootstrap.servers", kafkaNodes) + .option("kafka.client.id", clientId) + .save() + } } From 2dd8d657e4059a2fedefaef18d0b161bd10f4540 Mon Sep 17 00:00:00 2001 From: dafreels Date: Wed, 15 Sep 2021 08:28:29 -0400 Subject: [PATCH 10/24] #252 Added MongoDataConnector --- docs/dataconnectors.md | 35 +++++- .../acxiom/pipeline/CredentialProvider.scala | 9 ++ metalus-mongo/pom.xml | 6 ++ .../src/main/resources/dependencies.json | 5 + .../connectors/MongoDataConnector.scala | 100 ++++++++++++++++++ .../acxiom/metalus/DependencyManager.scala | 3 +- 6 files changed, 153 insertions(+), 5 deletions(-) create mode 100644 metalus-mongo/src/main/scala/com/acxiom/metalus/pipeline/connectors/MongoDataConnector.scala diff --git a/docs/dataconnectors.md b/docs/dataconnectors.md index a274b912..9f8b2bf1 100644 --- a/docs/dataconnectors.md +++ b/docs/dataconnectors.md @@ -112,7 +112,7 @@ val connector = GCSDataConnector("my-connector", Some("my-credential-name-for-se This connector provides access to BigQuery. Below is an example setup that expects a secrets manager credential provider: #### Scala ```scala -val connector = BigDataConnector("temp-bucket-name", "my-connector", Some("my-credential-name-for-secrets-manager"), None, +val connector = BigQueryDataConnector("temp-bucket-name", "my-connector", Some("my-credential-name-for-secrets-manager"), None, DataFrameReaderOptions(format = "csv"), DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) ``` @@ -120,7 +120,7 @@ val connector = BigDataConnector("temp-bucket-name", "my-connector", Some("my-cr ```json { "connector": { - "className": "com.acxiom.gcp.pipeline.connectors.GCSDataConnector", + "className": "com.acxiom.gcp.pipeline.connectors.BigQueryDataConnector", "object": { "name": "my-connector", "credentialName": "my-credential-name-for-secrets-manager", @@ -129,6 +129,33 @@ val connector = BigDataConnector("temp-bucket-name", "my-connector", Some("my-cr } } ``` +### MongoDataConnector +This connector provides access to Mongo. Security is handled using the uri or a _UserNameCredential_. In addition to +the standard parameters, the following parameters are available: + +* **uri** - The name connection URI +* **collectionName** - The name of the collection + +#### Scala +```scala +val connector = MongoDataConnector("mongodb://127.0.0.1/test", "myCollectionName", "my-connector", Some("my-credential-name-for-secrets-manager"), None, + DataFrameReaderOptions(format = "csv"), + DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) +``` +#### Globals JSON +```json +{ + "connector": { + "className": "com.acxiom.metalus.pipeline.connectors.MongoDataConnector", + "object": { + "name": "my-connector", + "credentialName": "my-credential-name-for-secrets-manager", + "uri": "mongodb://127.0.0.1/test", + "collectionName": "myCollectionName" + } + } +} +``` ## Streaming Streaming connectors offer a way to use pipelines with [Spark Structured Streaming](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html) without the need to write new [drivers](pipeline-drivers.md). When designing pipelines for streaming, care must be taken to not @@ -154,7 +181,7 @@ val connector = KinesisDataConnector("stream-name", "us-east-1", None, Some(15), ```json { "connector": { - "className": "com.acxiom.aws.pipeline.connectors.S3DataConnector", + "className": "com.acxiom.aws.pipeline.connectors.KinesisDataConnector", "object": { "name": "my-connector", "credentialName": "my-credential-name-for-secrets-manager", @@ -185,7 +212,7 @@ val connector = KafkaDataConnector("topic-name1,topic-name2", "host1:port1,host2 ```json { "connector": { - "className": "com.acxiom.aws.pipeline.connectors.S3DataConnector", + "className": "com.acxiom.kafka.pipeline.connectors.KafkaDataConnector", "object": { "name": "my-connector", "credentialName": "my-credential-name-for-secrets-manager", diff --git a/metalus-core/src/main/scala/com/acxiom/pipeline/CredentialProvider.scala b/metalus-core/src/main/scala/com/acxiom/pipeline/CredentialProvider.scala index 27297a29..1b2d6a63 100644 --- a/metalus-core/src/main/scala/com/acxiom/pipeline/CredentialProvider.scala +++ b/metalus-core/src/main/scala/com/acxiom/pipeline/CredentialProvider.scala @@ -19,6 +19,15 @@ trait Credential extends Serializable { def name: String } +/** + * Creates a credential by parsing the parameters username and password. + * @param parameters The map containing parameters to scan. + */ +case class UserNameCredential(override val parameters: Map[String, Any]) extends Credential { + override def name: String = parameters("username").asInstanceOf[String] + def password: String = parameters("password").asInstanceOf[String] +} + /** * Provides an interface for parsing credentials from a map */ diff --git a/metalus-mongo/pom.xml b/metalus-mongo/pom.xml index 4252d0e0..7844fa82 100644 --- a/metalus-mongo/pom.xml +++ b/metalus-mongo/pom.xml @@ -19,6 +19,12 @@ ${parent.version} provided + + com.acxiom + metalus-common_${scala.compat.version}-spark_${spark.compat.version} + ${parent.version} + provided + org.mongodb.spark mongo-spark-connector_${scala.compat.version} diff --git a/metalus-mongo/src/main/resources/dependencies.json b/metalus-mongo/src/main/resources/dependencies.json index ad5e1a1e..dbb23895 100644 --- a/metalus-mongo/src/main/resources/dependencies.json +++ b/metalus-mongo/src/main/resources/dependencies.json @@ -1,6 +1,11 @@ { "maven": { "libraries": [ + { + "groupId": "com.acxiom", + "artifactId": "metalus-common_${scala.compat.version}-spark_${spark.compat.version}", + "version": "${parent.version}" + }, { "groupId": "org.mongodb.spark", "artifactId": "mongo-spark-connector_${scala.compat.version}", diff --git a/metalus-mongo/src/main/scala/com/acxiom/metalus/pipeline/connectors/MongoDataConnector.scala b/metalus-mongo/src/main/scala/com/acxiom/metalus/pipeline/connectors/MongoDataConnector.scala new file mode 100644 index 00000000..bc5a17ce --- /dev/null +++ b/metalus-mongo/src/main/scala/com/acxiom/metalus/pipeline/connectors/MongoDataConnector.scala @@ -0,0 +1,100 @@ +package com.acxiom.metalus.pipeline.connectors + +import com.acxiom.pipeline.connectors.BatchDataConnector +import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} +import com.acxiom.pipeline.{Credential, PipelineContext, UserNameCredential} +import com.mongodb.ConnectionString +import com.mongodb.spark.config.{ReadConfig, WriteConfig} +import com.mongodb.spark.{MongoConnector, MongoSpark} +import org.apache.spark.sql.streaming.StreamingQuery +import org.apache.spark.sql.{DataFrame, ForeachWriter, Row, SparkSession} + +import java.net.URLEncoder +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer + +case class MongoDataConnector(uri: String, + collectionName: String, + override val name: String, + override val credentialName: Option[String], + override val credential: Option[Credential], + override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), + override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) extends BatchDataConnector { + + private val passwordTest = "[@#?\\/\\[\\]:]".r + private val connectionString = new ConnectionString(uri) + + override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { + MongoSpark.loadAndInferSchema(pipelineContext.sparkSession.get, + ReadConfig(Map("collection" -> collectionName, "uri" -> buildConnectionString(pipelineContext)))) + } + + override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { + val writeConfig = WriteConfig(Map("collection" -> collectionName, "uri" -> buildConnectionString(pipelineContext))) + if (dataFrame.isStreaming) { + Some(dataFrame + .writeStream + .format(writeOptions.format) + .options(writeOptions.options.getOrElse(Map[String, String]())) + .foreach(new StructuredStreamingMongoSink(writeConfig, pipelineContext.sparkSession.get)) + .start()) + } else { + MongoSpark.save(dataFrame, writeConfig) + None + } + } + + private def buildConnectionString(pipelineContext: PipelineContext): String = { + val conn = if (connectionString.isSrvProtocol) { + "mongodb+srv://" + } else { + "mongodb://" + } + + val finalCredential = getCredential(pipelineContext) + val conn1 = if (finalCredential.isDefined) { + val cred = finalCredential.get.asInstanceOf[UserNameCredential] + val password = if (passwordTest.findAllIn(cred.password).toList.nonEmpty) { + URLEncoder.encode(cred.password, None.orNull) + } else { + cred.password + } + s"$conn${cred.name}:$password@" + } else { + conn + } + // TODO make sure this works + // Inject the credentials into the uri + s"$conn1${connectionString.getConnectionString.substring(conn.length + 1)}" + } +} + +class StructuredStreamingMongoSink(writeConfig: WriteConfig, sparkSession: SparkSession) extends ForeachWriter[Row] { + private var mongoConnector: MongoConnector = _ + private val buffer = new ArrayBuffer[Row]() + override def open(partitionId: Long, epochId: Long): Boolean = { + mongoConnector = MongoConnector(writeConfig.asOptions) + true + } + + override def process(value: Row): Unit = { + if (buffer.length == writeConfig.maxBatchSize) { + flush() + } + buffer += value + } + + override def close(errorOrNull: Throwable): Unit = { + if (buffer.nonEmpty) { + flush() + buffer.clear() + } + } + + private def flush(): Unit = { + if (buffer.nonEmpty) { + val df: DataFrame = sparkSession.createDataFrame(buffer.toList.asJava, buffer.head.schema) + MongoSpark.save(df, writeConfig) + } + } +} diff --git a/metalus-utils/src/main/scala/com/acxiom/metalus/DependencyManager.scala b/metalus-utils/src/main/scala/com/acxiom/metalus/DependencyManager.scala index bcac9496..4a0987f8 100644 --- a/metalus-utils/src/main/scala/com/acxiom/metalus/DependencyManager.scala +++ b/metalus-utils/src/main/scala/com/acxiom/metalus/DependencyManager.scala @@ -46,7 +46,8 @@ object DependencyManager { new File(file) } copyFileToLocal(srcFile, destFile) - cp.addDependency(Dependency(artifactName, artifactName.split("-")(1), destFile)) + cp.addDependency(Dependency(artifactName.substring(0, artifactName.lastIndexOf("-")), + artifactName.split("-").last, destFile)) }) // Get the dependencies val dependencies = resolveDependencies(initialClassPath.dependencies, output, parameters) From 6fdf57977fd0dd4dc335d9d7c44827fc59d66a58 Mon Sep 17 00:00:00 2001 From: dafreels Date: Wed, 15 Sep 2021 09:20:31 -0400 Subject: [PATCH 11/24] #252 Optimized the DataFrame to Pub/Sub publish logic to allow batching. --- .../com/acxiom/gcp/steps/PubSubSteps.scala | 24 +++++++++++---- .../com/acxiom/gcp/utils/GCPUtilities.scala | 30 ++++++++++++++----- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/PubSubSteps.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/PubSubSteps.scala index 1792bf2f..0ba68493 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/PubSubSteps.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/PubSubSteps.scala @@ -1,11 +1,16 @@ package com.acxiom.gcp.steps import com.acxiom.gcp.utils.GCPUtilities +import com.acxiom.gcp.utils.GCPUtilities.getPublisherBuilder import com.acxiom.pipeline.PipelineContext import com.acxiom.pipeline.annotations.{StepFunction, StepObject, StepParameter, StepParameters} import com.google.auth.oauth2.GoogleCredentials +import com.google.protobuf.ByteString +import com.google.pubsub.v1.PubsubMessage import org.apache.spark.sql.DataFrame +import java.util.concurrent.TimeUnit + @StepObject object PubSubSteps { @StepFunction("451d4dc8-9bce-4cb4-a91d-1a09e0efd9b8", @@ -18,9 +23,9 @@ object PubSubSteps { "separator" -> StepParameter(None, Some(false), None, None, None, None, Some("The separator character to use when combining the column data")), "credentials" -> StepParameter(None, Some(false), None, None, None, None, Some("The optional credentials to use for Pub/Sub access")))) def writeToStreamWithCredentials(dataFrame: DataFrame, - topicName: String, - separator: String = ",", - credentials: Option[Map[String, String]] = None): Unit = { + topicName: String, + separator: String = ",", + credentials: Option[Map[String, String]] = None): Unit = { val creds = GCPUtilities.generateCredentials(credentials) publishDataFrame(dataFrame, separator, topicName, creds) } @@ -74,9 +79,16 @@ object PubSubSteps { * @param creds The optional GoogleCredentials */ private def publishDataFrame(dataFrame: DataFrame, topicName: String, separator: String, creds: Option[GoogleCredentials]): Unit = { - dataFrame.rdd.foreach(row => { - val rowData = row.mkString(separator) - GCPUtilities.postMessage(topicName, creds, rowData) + dataFrame.rdd.foreachPartition(iter => { + val publisher = getPublisherBuilder(topicName, creds) + .setRetrySettings(GCPUtilities.retrySettings).setBatchingSettings(GCPUtilities.batchingSettings).build() + iter.foreach(row => { + val data = row.mkString(separator) + val pubsubMessage = PubsubMessage.newBuilder.setData(ByteString.copyFromUtf8(data)).build + publisher.publish(pubsubMessage) + }) + publisher.shutdown() + publisher.awaitTermination(2, TimeUnit.MINUTES) }) } } diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/utils/GCPUtilities.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/utils/GCPUtilities.scala index 6c8d9128..6c1af7f4 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/utils/GCPUtilities.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/utils/GCPUtilities.scala @@ -2,6 +2,7 @@ package com.acxiom.gcp.utils import com.acxiom.gcp.pipeline.GCPCredential import com.acxiom.pipeline.{Constants, CredentialProvider, PipelineContext} +import com.google.api.gax.batching.BatchingSettings import com.google.api.gax.core.FixedCredentialsProvider import com.google.api.gax.retrying.RetrySettings import com.google.auth.oauth2.GoogleCredentials @@ -16,8 +17,13 @@ import java.io.ByteArrayInputStream import java.util.concurrent.TimeUnit object GCPUtilities { - - private val retrySettings = RetrySettings.newBuilder + val requestBytesThreshold = 5000L + val messageCountBatchSize = 100L + val batchingSettings: BatchingSettings = BatchingSettings.newBuilder + .setElementCountThreshold(messageCountBatchSize) + .setRequestByteThreshold(requestBytesThreshold) + .setDelayThreshold(Duration.ofMillis(Constants.ONE_HUNDRED)).build + val retrySettings: RetrySettings = RetrySettings.newBuilder .setInitialRetryDelay(Duration.ofMillis(Constants.ONE_HUNDRED)) .setRetryDelayMultiplier(2.0) .setMaxRetryDelay(Duration.ofSeconds(Constants.TWO)) @@ -112,15 +118,25 @@ object GCPUtilities { * @return A boolean indicating whether the message was published */ def postMessage(topicName: String, creds: Option[GoogleCredentials], message: String): Boolean = { - val publisher = (if (creds.isDefined) { - Publisher.newBuilder(topicName).setCredentialsProvider(FixedCredentialsProvider.create(creds.get)) - } else { - Publisher.newBuilder(topicName) - }).setRetrySettings(retrySettings).build() + val publisher = getPublisherBuilder(topicName, creds).setRetrySettings(retrySettings).build() val data = ByteString.copyFromUtf8(message) val pubsubMessage = PubsubMessage.newBuilder.setData(data).build publisher.publish(pubsubMessage) publisher.shutdown() publisher.awaitTermination(2, TimeUnit.MINUTES) } + + /** + * Generates the builder used to create a publisher for Pub/Sub messages. + * @param topicName The topic within the Pub/Sub + * @param creds The credentials needed to post the message + * @return + */ + def getPublisherBuilder(topicName: String, creds: Option[GoogleCredentials]): Publisher.Builder = { + if (creds.isDefined) { + Publisher.newBuilder(topicName).setCredentialsProvider(FixedCredentialsProvider.create(creds.get)) + } else { + Publisher.newBuilder(topicName) + } + } } From 189947bc79630bfdd484004294419003ca02ce75 Mon Sep 17 00:00:00 2001 From: dafreels Date: Thu, 16 Sep 2021 09:56:30 -0400 Subject: [PATCH 12/24] #252 Implemented the FileConnector implementations --- docs/connectors.md | 15 +++ docs/dataconnectors.md | 10 +- docs/fileconnectors.md | 113 ++++++++++++++++++ docs/images/Connectors.png | Bin 0 -> 139850 bytes docs/images/DataConnectors.png | Bin 47323 -> 0 bytes docs/readme.md | 4 +- ...DataConnector.scala => AWSConnector.scala} | 4 +- .../connectors/KinesisDataConnector.scala | 2 +- .../pipeline/connectors/S3DataConnector.scala | 2 +- .../pipeline/connectors/S3FileConnector.scala | 35 ++++++ metalus-common/docs/filemanagersteps.md | 9 +- .../pipeline/connectors/Connector.scala | 23 ++++ .../pipeline/connectors/DataConnector.scala | 15 +-- .../pipeline/connectors/FileConnector.scala | 17 +++ .../connectors/HDFSFileConnector.scala | 20 ++++ .../connectors/SFTPFileConnector.scala | 45 +++++++ .../pipeline/steps/FileManagerSteps.scala | 17 +++ .../pipeline/connectors/GCSConnector.scala | 11 ++ .../connectors/GCSDataConnector.scala | 10 +- .../connectors/GCSFileConnector.scala | 38 ++++++ 20 files changed, 358 insertions(+), 32 deletions(-) create mode 100644 docs/connectors.md create mode 100644 docs/fileconnectors.md create mode 100644 docs/images/Connectors.png delete mode 100644 docs/images/DataConnectors.png rename metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/{AWSDataConnector.scala => AWSConnector.scala} (76%) create mode 100644 metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3FileConnector.scala create mode 100644 metalus-common/src/main/scala/com/acxiom/pipeline/connectors/Connector.scala create mode 100644 metalus-common/src/main/scala/com/acxiom/pipeline/connectors/FileConnector.scala create mode 100644 metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSFileConnector.scala create mode 100644 metalus-common/src/main/scala/com/acxiom/pipeline/connectors/SFTPFileConnector.scala create mode 100644 metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSConnector.scala create mode 100644 metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSFileConnector.scala diff --git a/docs/connectors.md b/docs/connectors.md new file mode 100644 index 00000000..c8fe0057 --- /dev/null +++ b/docs/connectors.md @@ -0,0 +1,15 @@ +[Documentation Home](readme.md) + +# Connectors +Connectors provide an abstraction for working with data. Connectors are designed to be modeled as JSON to easily include +in [applications](applications.md), [pipelines](pipelines.md) and [steps](steps.md). Each implementation implement +functions that will be used by steps to work with data in a generic manner. + +![Connectors](images/Connectors.png) + +## File Connectors +File Connectors are designed to operate with a single file. These connectors provide access to [FileManager](filemanager.md) +implementation for the specified file system. More information can be found [here](fileconnectors.md). +## Data Connectors +Data Connectors provide an abstraction for loading and writing data. The provided functions work specifically on DataFrames. +More information can be found [here](dataconnectors.md). diff --git a/docs/dataconnectors.md b/docs/dataconnectors.md index 9f8b2bf1..dd6c9cea 100644 --- a/docs/dataconnectors.md +++ b/docs/dataconnectors.md @@ -1,11 +1,9 @@ -[Documentation Home](readme.md) +[Documentation Home](readme.md) | [Connectors](connectors.md) # Data Connectors -Data Connectors provide an abstraction for loading and writing data. This is useful for creating generic pipelines that -can used across providers without source/destination knowledge prior to runtime. Each connector has the responsibility -to load and write a DataFrame based on the underlying system. Below is a breakdown of how connectors may be classified: - -![DataConnectors](images/DataConnectors.png) +Data Connectors provide an abstraction for loading and writing data using the [DataConnectorSteps](../metalus-common/docs/dataconnectorsteps.md). +This is useful for creating generic pipelines that can used across providers without source/destination knowledge prior +to runtime. Each connector has the responsibility to load and write a DataFrame based on the underlying system. **Parameters** The following parameters are available to all data connectors: diff --git a/docs/fileconnectors.md b/docs/fileconnectors.md new file mode 100644 index 00000000..31c1f374 --- /dev/null +++ b/docs/fileconnectors.md @@ -0,0 +1,113 @@ +[Documentation Home](readme.md) | [Connectors](connectors.md) + +# FileConnectors +File connectors are used to invoke [FileManager](filemanager.md) implementations that can be used with the +[FileManagerSteps](../metalus-common/docs/filemanagersteps.md) object. FileConnectors are used when a pipeline/step +needs to work on a single file without a DataFrame. Most common operations are expected to be the [Copy step](../metalus-common/docs/filemanagersteps.md#copy) +and the [Create FileManager step](../metalus-common/docs/filemanagersteps.md#create-a-filemanager). + +**Parameters** +The following parameters are available to all file connectors: + +* **name** - The name of the connector +* **credentialName** - The optional credential name to use to authenticate +* **credential** - The optional credential to use to authenticate + +## HDFSFileConnector +This connector provides access to the HDFS file system. The _credentialName_ and _credential_ parameters are not used in +this implementation, instead relying on the permissions of the cluster. Below is an example setup: +#### Scala +```scala +val connector = HDFSFileConnector("my-connector", None, None) +``` +#### Globals JSON +```json +{ + "connector": { + "className": "com.acxiom.pipeline.connectors.HDFSFileConnector", + "object": { + "name": "my-connector" + } + } +} +``` +## SFTPFileManager +This connector provides access to an SFTP server. In addition to the standard parameters, the following parameters are +available: + +* **hostName** - The host name of the SFTP resource +* **port** - The optional SFTP port +* **knownHosts** - The optional file path to the known_hosts file +* **bulkRequests** - The optional number of requests that may be sent at one time +* **config** - Optional config options +* **timeout** - Optional connection timeout + +Below is an example setup: +#### Scala +```scala +val connector = SFTPFileConnector("sftp.myhost.com", "my-connector", None, None) +``` +#### Globals JSON +```json +{ + "connector": { + "className": "com.acxiom.pipeline.connectors.SFTPFileConnector", + "object": { + "name": "my-connector", + "hostName": "sftp.myhost.com" + } + } +} +``` +## S3FileManager +This connector provides access to the S3 file system. In addition to the standard parameters, the following parameters are +available: + +* **region** - The AWS region +* **bucket** - The S3 bucket + +Below is an example setup: +#### Scala +```scala +val connector = S3FileConnector("us-east-1", "my-bucket", "my-connector", Some("my-credential-name-for-secrets-manager"), None) +``` +#### Globals JSON +```json +{ + "connector": { + "className": "com.acxiom.aws.pipeline.connectors.S3FileConnector", + "object": { + "name": "my-connector", + "region": "us-east-1", + "bucket": "my-bucket", + "credentialName": "my-credential-name-for-secrets-manager" + } + } +} +``` +## GCSFileManager +This connector provides access to the S3 file system. In addition to the standard parameters, the following parameters are +available: + +* **projectId** - The project id of the GCS project +* **bucket** - The name of the GCS bucket + +Below is an example setup: +#### Scala +```scala +val connector = GCSFileConnector("my-dev-project", "my-bucket", "my-connector", Some("my-credential-name-for-secrets-manager"), None) +``` +#### Globals JSON +```json +{ + "connector": { + "className": "com.acxiom.gcp.pipeline.connectors.GCSFileConnector", + "object": { + "name": "my-connector", + "projectId": "my-dev-project", + "bucket": "my-bucket", + "credentialName": "my-credential-name-for-secrets-manager" + } + } +} +``` diff --git a/docs/images/Connectors.png b/docs/images/Connectors.png new file mode 100644 index 0000000000000000000000000000000000000000..5808bc211ffebfa115da87c1a73c38591dbd49fb GIT binary patch literal 139850 zcmeFZXIN8d_ct093u9pb1(lH~s9_Yrf`F8u2r`U9aIm11ps2{uL_|tL!YGP}7!?Gi z1oty^#{1=Y6kpzMXTu=bsN=xU=_tuY2|NTWcrU z!QN)+FROon!C*^m|FHTC23uSMgDrTnXaTs=^WE>iwc2wir2T6b>Y6K0A{)&+ zclhkVMXxU$3_6(WezwAO-nLg7%I%h0=5NuPtFpq!0kLh-$D)&w_cRouEd}HIue&=^ z#+?5WS^sLOcK6ca2frTe%{cN)LT`hmx@%nl`>Ch!KB|F&@oY%JG?;0A@6Fo5Ps0Sp z5zpbcb(z+403HAH`5zAa4+s7qabW9_%rsNVi8F@p#p)|(o``U}7F2&gM``9l`Q!il z3;jJqY#B#sN1ofp_=Pi1x}z2`|6{5D-(RRIG9rbSj#iey4fHho^0q%Pn_(ABbkhNs zB5F{={dPMbvn%M!hsw=~BtI9)mmS1ZMdqw`yN7%r9@{H8Z?vW`v~Pl|FA)DEuUCr9 zDtHat)wC=?_*UPrfqXHP+dnbEa9FS_64eGYK4Cc<>;uprLNc(35JWliur5;A~cN2vSo79?F zs8N4s*oYZf@bCd@goEbnK0}I3zKsr%xCzZKyw*stkbSDOTj)n0pSYmhRWP4D8V2aI zVRHM1M^`I1*x3)EEb34!Xb~o`BA@ zP?uMs*7eX4h*d*}5hNC((WcN%y-NT5@$hDER%VU*u_y33SXTqI?zum4NW7jr0lBu2 zT9Jo#9Y*9a}JKEW+B=$W73gJ2l{qharjA+rW@iJ77wr~y6E*M zLLe7f{|lP@D?nC8mQ?_fK9dEGZAyf)Cp`j_Mw{1BUB2B)!2{%dCn-cn#@g(9jS z3={AcwI>$sOSDcQ61NlG9@TpBhAcs~M!Ys>Xmq~gulc}+dNJ&`p z&79PnCCnyLL8=ctv#BbuPbS>4PG0M`06|4@l<#ED3DZqOtC!mtpo3DceQNSv-Zsb` zjOIb!U`Gn6*dA+MwlC~F_PuDd^WlbeLY4I&=nKa4f0p_VOTj2uEZ^q6o8H6G37oF^ z@8q-BTDAW`!BtO$D@-W2*?e##ySeCx`@A0t2*ZXx!>lf<;`f%tufvNOg-3*G&*=L# z`)%ICNy-6O7i}_eA7?Zizwsx@T0>q1ce{dH^TZr|_z?QD!noHHxAln6Rj-?z$i9PK zoI$Tq<<{U(Tq!H0)I#+qebS02UetlF3pF#eKcSDhNOD$S1>6C=)^b6ia0FF)t>duv z4gI|WgA4w@4D>CZ$mgq)4yv=OUiXz<@4q6a;pf&oC_Ox%tNfzslQHi8U0Sq*3>a%7 z`Ao_ov|HAe+@H-S#erTYe~1Iu|L4O0=lPMlkORMW|NcnxWFo`wIdkUQyC2~)T^;ww z*)zD^Af5U4uD&kZ_=o2^a2>9abRXV04;CRhw<8d~?8oJ}-l)hk0}IDsux?~$U1!t} zM_>N9-(sssf|i~*(^4yr0P!q`qQsaO!`Ib?-4~tv@#y%@2c`F+Anjf#I=789`6u*E z`a;Xl9jYK?l%M1GoOGM~6S}@zDUx#raxf7+-`-V}PW}m9kIk=}Jh#e~kLPv*bY zB;SXE>D3Y?b#Bt+pU?wgYvYY}Kwf7_FkEk7=j5Nz^|9SL{%0VITKNs>Qs+JSCv^SG zKT`_5wKKf%9&JY18|VEWbpZARKhwgkwVi>S2<`KbNm(=#i~Mgpi;;*NG!o%fFlb&T zYrys;T8IYo%XnFii_6lcQ!Mv?e-c4e*|jv(KNzNLSm@ zP}&m@*U?E~4C^R&+-=A2cGEe_Wfgg$-^4JVjtmZtEToLSu4G~P-``|c?aQT)uMQ~F zQfz^IAj|3invauyZwc(kQD+dUY+TT3tKoaFDVBKg158c~GgViADyBpn{7Ip1278^8 zc0PssF79UFJ?Vpy6Cw6Z-u;NiMDLj9C@g|tFgSoutU@lCF-^- ziC4J+Dwr;AZ~yp1kFhj#-j`o{t9YM=Sp+)@jcFw^%VD2?ST@7zbMn?f56}a?9;sAC z;V|`mD+fc%+xooq%585Y|8bJ{?2Gx_agIp2zKYfxX1!MdPV*}93yU42jkb{N zePg8mTY|`#iS8rrngOAJ5t#-0nA9=RAT2p3b)R@ttoH&>Hr^tW!f_ZbNPmX3hxj7B zX@P%K{z;j<1A4=5D=-f$hvJ@LT+eC-UU9B8WWPRMMZJ>{OCPMP!XiO?pmQLkLLpySxX7tq1I7^!&dy|8efu&mI& zi$A1x0-y35oEEzlS+!x!wpubz5L6JiQ^-Ey%=WSC)JLowHEkP zgItuhFylM!KH6Knx;MPP2aD|@E}YK1o}2xwmud&Y4hDV->-e;Y{4G*w{L$QE6@H^6$Z}zEp_%y_)s23XRODW4QOM`g^ubm~Y2#NwnVvnZ2^o}J~8t{oi-3xivtS}HM zT_yjHhNX0pzi*0Zdn|Ye^L1ZsFEPfd_Do9 zZSZS8uzz1Z-#3#KT(s}~kibK$oGO#8X?NRfkxh15+mAv$Vi#Wxv>J3^hv%fPg7PHiD;0x0<+e=cfidiid~1HKDTtz1EVI8tVA<%x143Q5x*{{q z8kcQ(J5p?HZeB#bG0>#?y{UqxEzK#m7v7uThfssqh+b|+gc;qtl$C*Yba}M@9AgaG zyYmxYnKGV1dv4;2L=uXrSuSGh%f3IunT2}e+0aoFA5E1`z+gGp#5})>?xV8JQ6tR9kDvaHXB!2?uvV)EZOMMnM8k3~jQQK%L_1DZfYTYdm0SqP#X#H@G(5H8JkW?b} z>BR)pkBOMSPaHNZwtau^@FR(61++B{6(?_yeJui2T2 zD5G8ZCh&GG<RixY|odCb1T>&R+ew6L?cd+rL0o`BVPNFpnphn1>_w zqYjf6*VO)-naK(W)2yM*=PRGfBzT*yD*IOU_@7YC2bqj46A4-GDiz+r%Zp}JV6JUZ zsj2dO^?~gx#BqKlf3OTid{_)Qm)rlDoC|s^1tJcDAei+UY*Ix zOf`~(C6W09uL+4^`hz>Q^axK%{RX2sgQ8<&dB--!qqHR%6c-0x;yg|hyNREa_P75e zJI?(~PogFO{2-cZitwXniAs%rSbND2 zt@ME0%TEP|KOOQ4){PLH_!sS3gjowOa>Du@`>^>zT-#Z3y*E4XRI*zKVlNULBA_?(L9G3)xL z<5|mVsxX@=>g-+i4Wtr#Gs84E(J`eH9efGndJayrw@56sZ^k?q)h-=mB_M-8J|X{F zGpT_K|4UoC60UXtQG`U&53dV}{4B8L9R3)h>CT#iJ?wxEG7)aN!g;Z%sg_;kB|c0% z7&&S;ir0K|G~7_Ywq2c@$$s%Iu=Al-)2PAMT*%Le8dmfc0_3rXg zKnI`1N<*u-y@FBRm>`Jufw5PBiAQdX?e0rrvf#?$E<0r6ENP;`FH^EQ_n#U}Y1N9% zx!yCZhPXwu=J?J5*w93f^7ykWoYT6*s;@yrsun_UbW88J9$xXm5KWC@6hzqklJeCd zsJHZoE}EA8r!Jzg%15i7_Iahdoyx4W5xWHmL%Otd31`hLdP+-=IQqaD`l$V+n4+d9 zZzui1KsBZXp*W zlV0F_@<)Uu%DJaaT)gNL+Yrvk?+T;LWRrUor@DTV2v${@I)Q3L4xSJN;F}8laT4d` ze=zTdYD+77Oxb>=(rk2DP+yNo)9c27_9cG|F~+TF8ezzN`4dLg=c2&&9Sv*t6ZgiZ zzT+CVna>qjqqQ%IyB0BXAK ze+C)Y>jh?5lIa_B*T~_)z~0q%i0aJSSaG1jXiTLj^ae4Ct_v7Z(t3z z_c329x{?Nt@O`HynSHx{H(3uy_265h&Jl*IxLRMR#F7I#L>V)}j67SF>i$%fhNzvu z-cA|Q%*a%IHvGr#2E1OblbO9AyKm?9HEB+a zSEu(1x~}nO01S-%&(-t_^L<|rlLC4^XKhxlsTve81D7#E3oI7TlspT6qRbeqcC$6h za(sfnZm)nr9xJJj<2JEML$Kedm!`p%%Aa9~1_$z%lQLcr!v~q+TN2PKvbPk65Se^5 zzUNbm=(;MmDX!mxAr=;hcTKnR{sX7WTvcOpHVsMk8w}JsgrX$w`x+j}iP5!BAgB&V zW*<8Hb6LKsh6GO@3DcpRy+KibAO;#v=kZTWAFLI0IA*b6`l#;1(OzzZndL;jj=84e z>xb*&QRu#kKCatd!85)Ze$HCWf{x6#8K`C4MLKWCj=ZuY+1z;mpmx!*$S}t;v^(sN zGh*Wgx$G`H6_rm`mvwsTu=$6kA;h?iEk8WzBtE=`s+STL1Bf zlnu}*uUECPs4GB@BAFTnhzPfS{sT6iM37B$tRvmNem^95xbBE-WIv^&a5Tn9FZ>xc zOe_5Mm2ptzmO^w*eqMdQd;~z*T}4C{?oIDdAL;jSf=}g$E0HqZUESg&0cQj-g5yAm ziz(hsvu!c+3THaXz#88w9S7L`MHcSQI#V(;Gr#J0F63)==2CcB1%4Q}l;6<&3GoYt zsaR6E)?%)BY`t)TB_93X6X36$4kvTJ`F;%5?Hym#u6>+f?Eok2p9cWS3C*5x&02M$ zjUb?FZ0EDN;%Dd3pr+=d-G3Y;Vu8(>JPmRB3K-0N7wEtFNLrt7zvO@1YDC?>tbNx9 zDvIRq963fo@{`*gM;8Ccd!R=n{`U54=)5CK=e53QQ^#%xmqsEvO~)0->=OmLiND40UyB$qgj{C6mJ%@GRXl^xl@|9{0Dj4vpf_7fpQi0Mj4}>Qdg*5?}z;R}B0GQJ@ zf<^c@2S9PmH9A5825USpW%JO}5i0^uhD*=8>d4;j@Kh2;f=fx$F#`HN_vDb4pf$Qp zyp8xPpynk1uw0`AwnWGN2KDWP>`$5GX`46+r3jiT_vM>%oBR8P0?#IR4ie&ma#beU zIk|q>#AC2B7&w^C=ySDh)}1lxw{S7P=revrHBaEt_B)p#c=g}++1Cvy`7G|GvGl8G zeF}oH;mK1U2RHJ5I`y1@^Pr9&R60To8Tk5!Bg$-H9sR^MmyQ+fMec-@;3pnN#0|5E zUE`OVd4juRB2*eO*uRMnWHF&6FRKM=nbjAob&@vba@X@9!YI&Ge68 zOLQ&lq>#K06HI|r6OejotR`T8g=zAr^7x@|8S}qP^RDK!eBl>Nq#rP60@Dj z=;8B8*unQgG1FQT)c(0}dB$zM1u-io?N^Fn@|q7s?xYkA0OiuX@=~7jJat##e(B}U z2o$lTpf(VdL&RScJ>_t{kp}@YWYb|290lf1SaZML?^HWfugT{vaP(H4+1Gn27Lc zgdJKgV$HShV#tQM@at(Q^%ko(-O)c>DLM&2A$RysT4U>-*Y?jMYL=E@g1)J=>e)54 zF3{Eksq!kn|5{V8_BGP=#-khV=to1FN%qOB>3k~}w^eB}Z&ZB)`grq%3$(&lguH28 zSdOT$adBHwO@yfC@%0nHOxrl{?vvF`x2&y%*!Txm)X*1n&k_Fip263z6>rI{QHwm( z(49yF@3|S_PVbVIs@1(&zsOL>E_fK#l+nyva(mC0kLwpLR5de>8F`Vrt@Zu;%C#r+ zLfq}n-`RGm`s7R9t5?_MM(xy%Q7V&s9D5Xc$Q7R^)E<2LjJB9un4PofE-w3E)_%4S zmt`Y`f?kwbVDz-~Y55^5&iw*ffQ16mZ~9%Pm7aZcJxK?$>;r8AAy+ zlKc@PG=YIA>mt+U(^JCp%+SY z+>Sja7-{s2!k>peEPojO94Hs(*BT9vX=Q;?ebp-Jqq;b!8r1}%5>Zx+_yr$xzTEy` z+5jJW5v{O^nNVGZivM?#f6`<;v-XQtkzn za)@o0^FR2nV770!>1kWb2rj`LM7ri9fbzh&wm;G15nfif?W~6oi*SuOiv^=EXazN5m6H(UfxJg;D-M+7Jm}UX4hnGE-XI zLsj0S?|9`3;J35d55zpw!*LdVx%$SBqQV#9Y~KT`J2kYYdV{b5w-rD&n1kiJ$o9U0 zr1Ol><}zLZZv4!nVc8hXpR&9pv0W~PwEKqNNq(&Q0?GxP1$Jf_AY#7uEq}s^4knisXMxBBsg29d2iOuzfxM9u8g|$%+0j+%b$n@J}>~awr6GahP+E#ZVl7+uu8IC&y9Pzj^-- zGm(dSa1wr0ss}~nVCU!2j+i^6Y3J%+g0(RdryPFMzePbRX~H{KjlI7jzVGwxhYY%( zEn;@4*C2i)N%1)Iuf@@grmua+9$5laW}X1|*>~yn@!^i| zQC(5Oj*UgB!W$-zGeS77bDL)$5Tbj_zaY;}*2Ov7NIaO3{ zxhG@7gV00p{pchVu&(jIa5BsZ%*RVs1u_aXAT^lR8ld$`ht z)>1(bMf3@yk&1kP4d07v@NGQS>>=2bxMSMr)8{wj-%)3_f7vblJ|X=keWvP767TX; zXEu>Lm_&*>g&hy#Q0>}ye@`8vQ(0I?vOoSQo5vZa*DhJi9H!9x9kETd<5xMCzoF;K zgv=p+CZEq4T~@3gDodU70LwNMoJj7iDtbQSmT2z44ZP?w$8Ohoj#PS1M4!pi@iTcL z5CnDsPwb`)?7HyJAE(@nb@=SYxaDTecxNdP4)FE_V8XMop1V`%8Z}BWox{n4!|Xii zHr95#N0Jl#(~U=jd-$0QHSCNmbM|Nr`b5E3S5aD#Xoe=i%I-eEH*|5;{$A_S~3C9qL0SIA8GOTJP;cfY}1X0Ul8@t z;j>5jBW`5$ux@It$%~4+h}}yt4ybA!PRPx z284V5V_0((wf@T@P>w=tfGSF4z|cgeohb=<=#`nD!Z~%_%=YBxtR;&EKrqYz?r<3s zFHL5rE&PUmDY5bN-~SuhRa!+6hV?m+HKV~HS-86G@g$2m?nw@09J+ETGYg?PW9?h_ z=2z_)VG$xo zJQ`HzT4%y85r(}ek!Gm5jxgT=*)|qme<*}$=$vL&jbZ7(1KThWF?@@ zXhLP)-S89#GP%P1MR=2OnbB3EwGOeH;uyn-vL5CYmhoRk517u=64I*|0E!w&{?$g) zrsRI$@NbmnErfj)qk#UW8E>#i^;D3QI3VKv!36Nr{13ED^cQp8cq()FV<@GO+`iGm zhbsgYO<&4yTfkuVJ@s#5FS*t;%+^k12V0!)2!%gtXqyft5$h^r`vAp9lIC-f3}BRe zMJ7u-o>VoMx^B-zt^=7daw-sOn2lIZWiqg${pwl*+&b}dU^rL#4KYs00@p6WP;uD{ zaW1!ZeK|D_rt%pfq>_ejm-@~$)!h&9gZW2QfA?2sI(GCG+1p(bVABS+7d=-61A)v9 z@kD|H@?&tO+GWIc;`(^%`kU-AA1}6%Vl$?ofVv%AI0?T9S1DwjPhIx_8)V8W$2XqZ zAxjUO$uZK;bQ&X_2l9OBhW7z7LKaey2R0e|KcGC=~~H)Dm`P*v+0}QGgkD zBN5|lU6k(_pSjlpUBa|Q9!&*rbLfJi(ze{qPJ@lM6xzSrO3|!83Xe0M!4Q|+_U-5m zQQL#m55E4c?*x$|Fhe=9x#}>O8qtC7!awC9&b?PgeuHbxz|0us>UQ1h$zZc>=NAY2 zYVa>tEexIvzl0V%juOJDdO{s`*_aq~5@b;KA~^%a8JgvI5cZFT#Ndg)IXQKpt;X8S$?)DBL+S%wCta-)bPLF*FW2Jk|` zP#&-qU(acz8g{h}oJX-swbUrKPgTitINvz!^sS>uT0th0EDk<+j#Y(MfPvQ5FfVJsP5%3$}b*7zCI0{ zmRg(x{ndthGN}cYr<+8g67akrpwd$>$_%~i5z-%Bc8(5Cwjwt}NvO|<{9T%QHF%VO z{ciFgy7HOyalHV{+^*E#?}c6~iPw#hop`Q$HqPmlZ7loWKfo!0vZ9Dt!YLMIkbaOW2eYHEEjA*FO81lL7zq$(X81*DM zm;U=(pV++9NYX_23Q+&1o6A4o5@%JY=$-77le1AdTu=YQm8|9qwb)AhBN*(QUzxE#ULG9DxXd7FbG&F(|Q;V6)}GAY~PAMm=ueV~=`gs63lP?xlzB4!>Zo5z$m;JD_6)20ns#?e6 zkEJA#;chjT-+1r(3-uYS_ao7rRZII+n-Ew#U2ktQT|Vv3LW0sa1BuBU(wekGATY+M z%I)s3qjoE@gO#e&xEZZs1q64lNS<5!?N%zeWZFIwV7tzTfgSHT&{t0Djr-F@egEXw z(BfI>bmjVX#t^TwRje7&%&Q1{tuRxY;1dBAJEmdzp%) z1ziHp0-%L66-l|Ja^<|3Gk4A+6YGOGyhg>ju-qH+R0aA&JoHgt!8FooK(7${R%xQs~CPozPh}nvK;FWJ+YUl2@Aq*u>JNo7WEx!`IxP%!^ z`Ug;AB1g&YVa2{VtMS>Lt-seaWuUbK+gwgK)3nPu3&`y118&HoWOb=;uNtAFQl z!mRu&R6}uiK`98m!zibk1ffq>@O4lX-FBkQN^sy=>}XTuXc5k@%4@DmtQ9+&QBV?v zsuZ?&Z9!q{7r@ee-xtqXrK7D=C_Wg!^LD?nM)Z|Y*vNRVEeaye(r}XV#mNo3mgI`z(31+Z@Ve8#?oDHzcmGT&Z?ybnnii--0nID16USqVoF39Q{ z%;oOTUHeF2lXXdQtKV~MK0;Txq&OeZp@vS|_>9^j@F502gSljGN+I?uqKir?O)%IK zIqi(t6`-l4HtBq~gZb#;)877mhyvE3IcM_}*ourRV22xERCor^cy(yS!i_%E0e-b% zC0rBnoH`)zNe4UVz6EWb(mQ`j^?0YYfiy)PwmE)s$|N_oB{};xgezB)nE5F>uv$(q z>s3!F0zm${nsS^xw^XJ7bgk%d0D8s5hG*0>T|TWrV22X*9x)UowvRKisieNcP9Szg z$sN9M?(lF$tk3gQV>(9p`{88&K#ctg*n%wc-+hP9(Sf+?;PRU&zU(nirmN! z`jiO1`ST;%p1c*<%KUJne9rOw7NANIIf_;~zP@c3;G$JjonMid89kI2qq*>Q#Qd~Ij4O{K)kXO{=H35V!<$-D1pPe!$93+ z%9i$Kd&)R@q?CHLbTH)k@DZBEg)X5lKbet+^VrD8-0|YPkbKFGqLL*8MAv(0sq3z0 ziQ!bj91(bUWn!**G6T`E5+9Octe4En`-lE?ZuvlqM6o9Cr+i{&#BUDl< z3np`~QrAskR=WeEn={&0c4&}y9fUdEWWzbZvd2AG_U4B5?|~}8@}QX~qg1P*=w^~h zJZe2fcw6aaxT&HzTyr|Fal12;-E`HrZ9DM^GNA0qQrNX2QH(7y#Bz+Y9z&) z!25rzyz_i%M{YvwM!k}L)2IJ^ffCr!JhC&D-Z5MT4@S%C^5A8>?_%Wo$J9y@$xP6@ ze@{4lAkcOS1JEskpgKvHq!|R*4=pWT1!V(}FdF+Yx3KJ6>^)~*1kqK;V6aMvVx^K1 z!12B|Q$}dtzgtGz^oWdPp5$Aq3q^_x?0Ba~Jd5OK<5ElOS#eYKkCT?-2bF&}=+qAI zZ7;vF@>TrmdwxFb%y@VWo&M(~zE3J~LKL5Ov>?=fxRiYu{F;F(=)on^FXq3a`5PtE z^-bfcR|P`RZ~4_Y2N$9*7fGCg+sfx=E%fZL7FPs{$g)=}C*@x^s?R%JQ0<^L~sqdgNf|1)#Z-X7p18 zsc?sv8M^E^%CEL1bQ0dZ!bx0JgE%J}PsX=|TtdL@Ng3*R%2+e@8A3128V*Yb-A%4?ji+~*6yG%X3cTAI ztWrn_V?drDZ)7z3d!SzGx;I$8;Tc7Cxh-t@r0}iDN7v4sc(3;9T$oEe-#@=$9xP^` zjs^UIxNsYWr2^&`fd0N6p`KU8#W~=d%orLChy<#HCs;pK4(RPMyoL4oPO%o@(3wxRp!<|i<7`L+d-8=s@x^)JkYIcf3`$Lxhe zdwGcCVe_?M?rLC&QIJv4rBO}7^QBl>NOoa9r+o|Esyq{A&WDN#PI3NNYxR0T**7yWNDy zHucO6!f7++iRde!Y~c#0?Q`0_(j>@3USY6PU69I>!;u_F-x9SX4KWTD{GA2F`ti%( z0w0DF_fLfur;xVC_tV3ll|OM#I}~uxu+bE(pL3k76h0bBeeG4yK%}hxM5kYmn;*ek zKk4xmhNC~}bHV$wwS=UDdni+AWqIc_L;sD^HV|_ieY#v!m(77S4za^Q>jDz&dPo#| z@<514^G`;tWf+(cgLKgKzzKWwOY9wFm0eWom}K=-=yA#NX+BVgtUpo@tX5+uSi^BX zFj0y{j(;jq9Q_EqT+Z(YLCFPl+z4^Ml7mxXrhK)UWxg|c5#CVqVr`u5Bce}dDgR4% zg71A$6%?DTGGK@XFx|G4C}c8=R?qJt6Ba~NfOj9K`%&Gin;)eC15_|q{F-;72`GAQ zT^RbV*Qac5^X46GCQ3iGY06jp>CWvQeUqwS6t}=MI{ZBtY1mrH?@7!JLKY03nv-58 zSMj*qq1z=i`MfybqW>NMe{E06M~MTr&x_n7(a+s_6k+Y44L+T|mETx@y$EzRl9tBB z`919fg^N#7-PIZp<*ougL}cA&Q@G(11`O&^XI2(_(7bI0EJrS(iTHheMs+^l)~FKk z(DqtfIepv~bZ^w1S##WRa*6ax$9zls;?plRKNy4W@9Bp=^x?gTe(#S)@VsJy(9XT3eKe!zAV6wbOiW{QRkE(;@5_#RiqWt%^3$+DYM?jx5g zebG+V%4$8GQn(?J0%UO{2db#tcgePF<7RW+ZvO(S1QVMx0=VK6j;iD2I;Y*xAgzxK z0A^49vMtP08}XEs+BwarZGMoKY zv!cwsrBdh_)9aMH5};~1aX+Z@jl*a4-_7AXrU#@gGW)_ny6s#XVJ*i7r$ue~_daMR zt5mhUx>sufT)E9};8b;el*=H9Mo8tP5}2h~MNo}6B|2%?z!KVnmVndxW>8PuAYKHd zjl_3wo;kznwB^q3a)13z2GA)hEkkXvXzi1B6N_(~&+(9h#?Hf8+1#{{CA&N1^km@e zM1Z(wbOxvcutj&cAovd8seGlq+q(-CwssdjUg$fq;%uuA;L+AJA>jTd8ptKM>|HX$ zdEF**>mEN1@J;0?V*qZs=f4D9He_uJ=t0H#rJ^nn@qNq{6mZt=O>$4S{Gvhp)f?hBqPuXBH!dk$fUIh>s-5aHwT4S=g z-c5d~P*)rT=|;H%viq+?9eo8c9Ir>+J%ZB}qaay}@oxca==YJODWoULX>SH-GLU*F z6jbcNG>UuZ>0xaCdtleeW{;)O^V#f=R3(Cpwb0ui1#{P3@K3Mc>b=)(q)mO0w z=R|?o{R_j`9PeSX^7Ba(>}4J|?w`lbhkjNI<^=2WAwYc=`+NnHo!~-c&@pG0#B_yw zKy=VdQ@ipc24IyiUjg4MtvnU~$R31t8Lr;QiAQsVDLL&rV24vcpSL$rrZ8)?6S6~ig5E1GDDB}!Cj!VDx`^Rn@_-iV_ zZM9KRITc8MGJ?s67`xSHiS5^4f~xE&FC<}gLI*p5;LdWX4bII={b+STt^)T z$^c8SPSZ_mHFsvIQc#)=iJ1cac`Yn(cR2usp7s*yackWVEx9$qX^T&cC0U>okon)4e-c7PG!9k!NG(&q)|p3W*nxys*aAK;pz z2j+l=v4UgEjZ{k^NNCbeSz<}O3$}unlBT)l9&Mo5&z*;}xCb!Xh|x~H>qo)T=w20KNc`bHecmWd2eLfd7SU|zwFO^DRn-{G~UnO^cd6ELB=tdkMG@>>U z@x*gK_H_`hM2Bdv(!iS5oOBDh2O<=prqotAWNrH(2QjHGBDo{YtyU{KVR13W9{^b}5Vgqg5qRFR$+ zglqLse3SvwrKWEA|G;9_b1!>CLj*9F{Q8H5_q)vrHjv6)1&loWYYmmQJ`ycy z{Bv&KgSguMmlRPoe<8i)NykdC?0Z2I9W1)B$Y0>Z2$3wN;smd?iP}Fh!H9}ma3rK` z;rJ(F@ko9y5HO5Z4Y&<{PJ-o0J|}YYsA3VN6IBZ;IjQp%nZt2AY7#I+{i(-`cKXC% zac93qg2y5ZYgA=ybCW_4dFQFa4DN$;3#d{8AkhF+Mcdl$(FYZ~RiDag3&9H2Hw>1YSpRq1wKFl0Q_a0G1f2iM zY7njRUIQYD9YMtr4rB?pmbAuoQI(Bv9LfJUwQl=9fAGFq2MJAO-TaZzrZ=m+QEE>~ zh&_u-2l*CE_mR&vYQ}^aayv+Dd~&G1$M(T$ZFLw@+pqMe^Sk>LSa~-;cKlYu_oGvu z6RM~1+l-6AL|!wlzAD|~Oicmvv9NLxa6TK?fi9(Btjgk|W&~w?D9h7%;0i~bWrHa( z1Z5b+RaiUwP2XkH@PZG)wb~ZxP{>)CO$P2z1jY1_*lccY2qaIom?^!o$SiJ=xj(fM zRFxG7)eAq~Ae+<$B{$JNj&;$%1!&2QiIawT!yT9`B9;X*j!^5}`K$orSpu-SZT}ru2UG<|r*zMgP^-Q8ADD4l%eHnuE4R&TTpr^1F zP^H?%3Ru%)22eyXwqBod@xnSJ+%UQ#$~T!^=3_aY(PZ!(^r>Z!%DOz+ICe6Nl89zW zJUN)qJG&(uNJri~k54ri-3K`HK9}AIrodl@+4_TBBC;HPQ&AI0;D>ykQ+)vr_OQ{< zQXoi)>~4D~y=&nLU)8`m|LEjcK&ICiG6sM*MWP}qQq+YavjeSuz*UXQ6njM0oY3T@ zry#;!`tsO1Z2h5a)ij1z`nNM}Qh4Zw1+Yp#}~40v99z?^Cs0 zlpctC?R)w5Ta4jqkN6$N7BTm&b@0dd^;7y*`B$0v;z>Rn9qRrNQ)`Gw z!QHCxmj~7nlz14iXZMe$*oX7;GZ!7T(V-MVUNLl?F;%8sqw(XQ<&n=E$VsJR<4}`! z$a#P}Ebh+Z&-v)m0Ok2G-2`a-M`W&Pa8{)i@e0mrUi=Tx`VRBx_iWs^Jn9jPSD%b7 z!aX0A&J3DmZ2ht@ty#&4Mj2z`H{XF=zmq+LNkdIGsC8PIJiy%)N!15AJxHj}?AKv@ z6O1TPFzZ&Mw#rI~SbRV4{C%2RIPYx`NqgT6rMX*^#O{HaIx@zUYvA`a-d95gq?_n- zomx3zW*EO=aqK%t!b^E=huF(cgn^$brD~AyJ(k94OSz9u3h2Y@oo3c5UtZa!Fiy`9 zyd$IKorh(!8nhh=^6mK{`bE#WNoF;vlb-PAuKj1C=PC zkAM5D=0>>+ZeJlNJ6oOucX8t)u5l1xO`saqTCDz!{f*=Yb?{zn_-JMz|3E94Av<=3 zgENNcDL3Fu{G~PV{`Ia1~O2znt@8KYP06cC9}Ch z`J%i<%s>r$6|hu9tyWT0jQn?|6+NQ{x5XFv-=iI&7}Ndi+5mV9X=W3H!#a#DC_nXz z%}`|rdfRK0JdercKFS-HGP&<;ZbLB!<{JRK{7w?C5JbRab8=7!7R>i)zrK)Yi9ugMt9W;<a7#VHTVw|lNb8Ag^=3JglVpl%21==GFJ>~`7nqZr-56lCp z{c!`gp^{^uF}Nj3zV3`z$C+1p(Bg|~HAsp9BE*?+ird2V+>h|N&$>Q~bVRJnzkc4g zRcZX?&f3Y8q#VNuX#?Ytou_I`t6B;V$Yq|_PCO8E86D>=q3yrFgq*CVg~sdCND8s4 zds5u7$K-P2z5P+4;o9JA{gV;Y{S`FQ9BSr!Xe$K_2i_^CM{ip!zdBi86?A`n3Ly^} zFLxeNy3aMuu%UNEAXIrq72%pDcpih?qnHS832%RtXmjP&C$)&;T z;OwfeKIOtgK=1zaO;R~@>qvb4sA6x_LN3$mlECM2@X6YGkJQSQ7M{?T>37Ypow+dz zGlc#l@d2~uBj0-BP7p?Yz%(crdRMK}*Czz6_7JB)-X`J*XKm89%6YBO-AnSwZk^!# zu~wsxE`?^k#+tQw-57Jh~U;4s$>9D_)1oi1dU>e|>d%*gusO1*M_bkllogL25&?fklzn-0pKkzqD88F^4 zIdS#NcXWT6bqZ&^ps`i3&4T%mGYHU|toi^S_Ro6R!X=PGME5TAO{9*8PL?x3Jm9VW zscN|VbTNb$zMUXz3;^itQrOfjb+af-_scF&MCXHP2Vo_L>Y1fubbb{A3I-r=iJYP<-7`)wP{iu zf79eI9sq9%yi`{VB0GJDPu`uwxH60ub zG{6r&D`)2)(wBl52_T%o0$0cPVcQ;^?lKEy_z*7Q2#lj3z4$V5GBS5?d!@nR6HC z3HzepTaW5wVFiQDx|o4aP#+i9!BqVSsGD?;u-x~vvf};?RBIH3+A;7ubU%J^QFq(y z#%8tS6Ch<709aqsq>v&ufH#e&qOfgJf=%pZF7aL}+%gG9$*dImr4$TdlrV#vO#8g6%`{WzQ+L}d$pN|~O z*){10U{2uY@1_uq2y4T$d15fB^KKAY7FK>~HiqcF?64lK0d_EsM7{>92$}B9Vw|4# zvmYevnl(+Q3(3|2d1zf?5oCdErfL<0YOT+po;HMZPHS@QI^_pqOdy{s>usaRlw^ZG z((y6;PO14cDS}(|oM?u~Hv#nr@oCj4Z@NpAa-ZcOCE>pFshW5u<$#%mfQ2`wWLtfw z68e0|T!vJ1g}>t~Uk`I0&ZmrC&J#IT2xX%&GQn3Ft4}793GfQ7QxJfMafcd1^_{?;m$-p7WgNob#OhJYUT8RCtPH=$wn^z8_Is6V#gpCA6lXT7|;= z!$4qjSKA%Ov?L#!>R6c}tyc(A!tIb}%T=2jRx_hXYUN!BBv}DR%txjk?rtCSj2O#J z!3n2GF8tgTKSHOBiMP2Io#$}rVJ`H#op&DeB#w|@0A_wa@;+C zpr5DS5tofwd*fGMz|d2Bv8WH-@{0c|6P=&Lpe&xf{?2YVUC@3p1f$#f`r|}-XsbZ% z2%i-sEf8E#Zczwig))7E;O!WdwVjS?&Ke#L*bpzhl(F1cL!L*c0PmG07IWF>^>|sK zxMyf#YpS3?kko(~4ODJt9q&b=CfbK27d(>L?G!kxV^Anaa~2y?RZ&AfQbUP01`SY~ z^v`pF3o9H}ssbKZ8P6+#lXihuxZyl2pms$?E|_BIq&ikAiHHj6EB`L;T0x|U#Cd*#56!v3@O#6D zMO{qly}=NQ^ADbJAT_-GM~xgb=EdVI;_dFz*R9?c7;Ny7w|I~*neGhfj(y4a{-AoQ zG}N0@)>LH}&v{rS_&6%^$}*hICrK>d3o<-MTGK-?WI>e0oQ$F_Lm&x(r{CDmKQJ!x z(kuJS85fC#Y}Og`ZU6j8+l6HUjWPdb91swoZr}(l2C>VuFtCOlEX`2f%kwhAA0AFq zcV=s*MGmE{Qtn_Yj|;%K4JJ=!yxgTZBN$#O6*L_pir|a}&;Fnd?D!D~F{FHIh}*bjVX*fsA~s`pvDx!p&uNfp2?bL)ceuSz;bmRQ zevTac?k!94QQUUHNZCjw&pMdl*y@Lmnh-?@B$Qry|e*%JqOY~5_oU8J|!IIDA2P~SVnqT-kZc=ex)WT9$`~LVhSlB(|gbeil zdiSS64x(>mSPxbPTGhp+q{_YSoBlqRJFnTc6T~y+W2l<;b#3ghKl^8L5i zEeUa&MpYxriunc!J~Lg55j-+V=fgLYeTb9CocntTH*my^)U;+{V>4T})!TH!>ilC@ z2m2~-GsE#vWQ-)I`TGyRE#p`GCeQpVwrlLS4;5sf>8hBx?WRv9?}>1Fw$)sz+uJMl zp)}nI_F|?Zio$VD%B&TwwB%RGiP$>$vGo94i=*lfoL#s7@2rr5@HQAmJ6rNi#?Fg% z-v?RpMbb=io!J5VD!i3YS5>rVJ~inYB3to;pRj7_C5_%{@pg|uljAooJ~RN!VsVS% zV&-A^ahd62g=LNMdne$+aYk1N1OttnMrHhDPQRJgki7JdTxZ|EvJe=?7PG3H#(g1# zM-ri(82wyO#(shtX9we4XG;ePJnh9)>^J9I(oCdiFzZ%qNMd7+=o*gHX0lGlVmM4?b}1h$e(8-Ed?vE=*`Z$*S@uv<9M(c4b_5@=+ra+QLej z$z~OL+E<-VlCF54_rmrtZM zKlPfzRxJ|7-Xg(1SU~i@jhs;$IhRh`QqkMKt7^G}H#fBrl}#;_BPAgk+2_{gz+Yvc zx3dFFFB#582Ql6ykw=yir~^;7C!NT5^D5JcMl#B-35J!GE$7`xrQYYS$~XU17@EiI zL(72tMoug}PL-OBEG8~!j)8aK+WR|#@=mK}!=1l;7JFcYWaTou z4B~i!cvKUP5Mo6FhqTY$rdsf}cT`Qyw{CIFc=g#T_lrV;%X4nz?cg!45+BpCJN8t1 zeT$*)50Ni`$9hZ|{=8Q{e@in1`TgWj$1^pZBXTZL`7~W7d4lF3dze+|Cn-*!;HH>yqzhW5DkqNM+_!Ed zU3YTg1uQ?asH8ZxiuzKB=t5yysTYTDP&&8#i;T4)MoWKneAwg!#6Yi;VJIfo2${3^ z^@cwf+u8FcV=sQP40;*=Q;eCp9(edK{iUF;g^yqTL)-8+P=>sJ-5hWMwO{`DkG@KQ z%~V6{C9&;KEpf=!-eud?A5y~zY1*1uL*|6ZXMqEEc3 z{!`{QZ~eW0EJVJl^d`L@asqRW{0**U{6uTdpSR-is(8m^vh&)1A4#t&NR<3jprWHQ zDo!4o-}mX0;F$f}UdlQ2hR=r%a!3Bd!uno8{?A{*To?SePj&nf_V%C0hH32kzyJKH zg2S<7h4~QmDNN>e{7Jc`&{n7ZwmrSg-S#iY7`ZM``Rn8{V=X9MVSUf|#LIZleGC2y zll8GbvW=Mmg%dls4jo81IdotiO^s?x^_TItpirqg{{L_A|49T%%m0H?l#%|z;n-`d zgg{q-U!TEmg#m8PS;wCW!usBS*Abi{67^F*fQ$B@eu0JAYCtdEz?5>`v)k=L=jd=gL&D*N&eV}ouYa3mVY#daH|lQ+=j21#M9EjuoaqBe>A-Gg;fbGG zw>rjp0t)hXo~Z-ccUW+B4K)YJCu5mJHM8IudaRygxX*Gra$3?y$8Gq2W9nv56+7A{ zVk&BbB5JDS*_^EEPe40QP7jJjulWCF18A_Fi-8}b$99AhNlP@;{#I=*F^^3~nYX=r z8(V$_^8F6`h}LeWxz)lE;zi6~Ds0b&y-jJszLL-6q`-&sFDAO}Ow2$Nva&)#1Vrin zQ?l4H%R$0wEe6k|6hW5izL3qIA?Z73@^E=Iwjy5GsWx}w{(WnngZmE(fG~5U0>c?NN~6v z&UL=ltDEs^BvqQuHK#0z*&$*ocd|%WQR;-tb0pfb%_ISI?mMU08S%7R-YMqX%OW|1 zruN%1Z9MFegcIyFeQyO~fOHi>p7i#1^zyD-{pFV_dwg@Yfd4kV)6V?L5Th5ElIZzF zP2EIW1Q&6EANgG1#nZ;}@#w$u;JN#msBgsO^*jgBS*Ium-cc^Ww3D(Ka(X?_Qw_D!e1M9Q_>bYs zDBK`_2kK14Rpo|(?+EqG4U%Y2%vPx%1{vn+X?ICcj}XCqJ#Vd6K`F_;&a}VnEgS;4 zwyrlFlBQ<@WPJif70cbc*T?HoU%rVfBQEGE_oPm3v!Y z$(5s1Gslb5n`hjsE5DZ3S)_HjLa$-ZLqFXJ+s@XZuj5}bD)Q&Uyy)uJieXZFfqyc~E-YQ7P z^qsZYs)h3!ayKd;FRHv=UXCb__B3r|1q2CwRTEKb@I)$=J{hGlVB$(w9ORh1&sdJ5 zl1E0X8PkN0aE=4LUEso)yBMX9uDagC-ia+NJOR^FF3NFINStt~$|+sZsJX`_Y9h&@GLK+2SG02ORXf$D z62JNthbZcLW;yO|$+xk!7^Yb?7KmOsSPbRU7$#&bEXh`JmOJB475nzwsT%zm1;SDb z{GV`^Ho(1xfL;dTsK}&9da3x(9ce@C<|%nK1KfHr2??mt0E82Z<{z$n-!I+N`bL^0zFx|m6xzN}3jXgyki&kdp$aaEen_}9VzRnlYc!v<^t7}1?5WIe{|COUg!2$42Su0wc~lWPV%4zG?^ z%OTFaax1=OADUJ-ysYDAL=ICPS=K-e=a~R%cVPv{Ar=xnHCLQ37=>QSf>jq9easMY zMBiD4`)Sh{azWq%@b_&W>O7pK!D|?r*w6tn_S=MA+0s}l$!?aY{knwxfNaEP5Y{jPRv47z*F73WS8caz62K%f4D=FeZrzQ z$Mc=agY!3B+t)}nvK-QNeg!dkThsa!E=jcdEQmKhqXg0ef(Y(o0qQoBgcgsce})Qy zeFkYK?JX|!>(^sCRt@D_wV@opSGmqP9quD6*9=_5rLJhCs(uqJXvK$eTb0X>8***x z3OB^f1?igJ$QUa~7YtSr7c<$miZ|1(nr&o%5png!8k8|yYc)OeCQiDZq zb6xkVeb$BM-|8OMr_tioxhBzw02MfU37sl6nr|Pf9GY*%N4XMnr-WUo>?-!fu$f8r zBedh?isktp<^+=hYT6vpB^Dy^v@f`GRJVfWPHO*@8ObEu1vh3NYvYpTYd#vMqTlOr z+*UW+J5CTtf3-p4PJ8w!qYC6Q_1FRXQU!#OoINIVL(SxCwldq|fu1j1& zo<}Vle~>T8*OvQrS=F4n4^|~(Q(&STce7CODCZ1M@S%zp$``g*>Xe1%MSfS?+oX!0 zFCj+lNz4!u5?0E$Ey3NJ9}TfOeDbSkZWR!6b&x#wVikiSsG2DBn*&d!({; zxu~rzy#Ya-Wj=b#t|v(i3$gTFRV&n4tk-6jRPQziC#tiGo}-l$bxqsL5Dl$6DqL7X zo!C|jo_nUGr!05(oYUuUtP`vvmpfaI z@p&_aROCw!8#=kD#!Kc7Pf7gRF{|%j%~D7s&QbGjm?I=mVHndP5T%U1{gE~Rulf!4 ze;8`5@(HFLv?%NHFISUcQgMBDTG+0+7lH~$s7e`z`JI4(RUg?wC<~12BFVp?Ffb5b zJ(Ebs?JZW{vD2#Z7Ud^m(^-89Py)d9as>C-wx2GAQnJBD$%|wxZOuLkWlDgs8b(<1 zJxyA%lTB>OOl3a!J_vm6c~kqBRfPE;lX%) zXoc0vKUO@3(le{WkZT`dyU$n94gT%*sLw$2&llw{08^LXiNmT7s|`NMx}tClva(4iMX1Leyu0{crUdAY0~;)^TVQ%VQIc-g%dgoY8SP?Icx&Rl znZ`PtcsMp``W<5rt`m$hyXyMgsQq(0dr&R|nl#oO6qk+EgAu2qOA2vv_| zkYk+@`^SEATB@H(!83DHc#kP^JAHt~@uf-Ed%GdzRpY%&!ue(rhk-cSh(2V;_SyWG z^MS!a68m^i7E4Ec%AtQ&o<4xAstBR6>8kNKlIfsHp1Idy{Ywy%Dl)(GYp}7S`+K4M zV($Ll2hW=H`~GOWQbYlGe!fK@(DAG0OpdBQM~0sNStDLM5UYGDj_cQlI)|uZ21f z^`TwOBoBAFnxLnJv*$leKdk(WF?1Eyr6F_&D`3%oa5}7E`>n|o|A*M_PmGtXXp?5# z_u22#%u{aW+cu_&r=EGQenw%7fVYgBV?zyVKF9u`zIAQs``OYDvjdmJuE$hslK>@% zg>aI=9?9GCo5YELN8>}M@f)En|Hrb_ytIf($M(hlC}gL9on;2`9@^A0}9TJ->7jq z9GF3aaRWUgNGyE>e_$Hjk^g=clI6(E9}dP?>DY^i*Pb3-E(Tpy(kzH=UVoD1^N}P@ zWDE{mtFiUbBycH}KDW=^;!XvgQi;88IPb@K+BVleb*D9n)WDrCK|vtK{si$nUasfC zKHuGN#@o0uSE0|S+z6BOa6}Mim|t2bGg3ernb|gvX#xBX1?Pp>Un5)Wt!%V5nt^NE zWp^2f1?=bn%hM_ECmtZHA9~ftURI$D@lFl$_jQ>O zlXWjgwIp~lmbG`!esb9~;?`8@d=vNhA41)k8~Jtqw);2sYUrGgd`q$^<3|=3TfguB z@NROw8BWsw5R0sQb?7pNxR^y&u<{4*kw6W@zpqlYE0nc??1NKHg=~p8eOu^fn~+ua z6_-}I8@j}G39&i1XooI`$TqV$TQyZ)gjB>G1|VQ}HvgB*hPi6X;*o)Mp^pl@_7k0D z1|W9bR#9oSojNAv@*l6 zYu?{Rpz#7MsRy;hW~y1OLET!XMwd2v2(+<@kKR5uN#i9V@cSO~Ue1_rbu8UUfC%XM zhxJu?9offe)HHrz#eD#RaUNheXqh1n0-UdZnI=WMJMzATH{Gg&9j5% z+sS*1AQW2u5CNI^ut|H%*8J)#guPkI-afvh1ze2mG6aqOmeI-;$MG^#_Y_VGy4npA zcn39kVEO;>AI6SGd~grj6F!5mn)H0M$5NnUs_wA4$zpmD({K6~mnCbcyrY{}mHKw~ zziafnwcU(lAuqNw{Z#vFJFPNQi|WK z-;r2(SnkHwhuo#>dlexOaR1{B%Eyd$hf~xETeDW@=n*wE?XSiguLW!UdHMJN*SBXK z^Y5H+x#5--_aGntw$SKxy+9Kn;q!+rX?ecfccObyIbPX!!-pe&pMdG%U=6X3;^vu8 zy1v|lLGCf?`Sp*P?&XKTQ+Kdehkxq=pP3u`isfM5%5B!w7Ic=veZ$7+6RKx*KYTMD zc`ihl1S{{l6E8Q}l^YxTUj>XP>mV?w__ zSx`oQP~4-#(DK_pnt=}ba=4VIj9muYxGq$nhEWb=Don(bBKJ-|~x4Pu~9P}`W_ z1uLT~vZpnqlXqTJou%g=7YgYpEwQCZOO`_AiJ|x%%YcC5?bh%5Tjd<);3EWH`bk`R z1qqnf>RH+Uid$iEKA_CJE|2Wf?rB8+R5Yb@=-KNA#wnz|1vk)lfM??8C*7?bHE3SZ zd&scz?<3;HXy}^2tZ!BChW-gF%bD1ax^ZA4?JFa2l`W5NvSj(GL{c$GrGWBp*@*$A z6#@F~(??HR=s~}>jr|$o$;7Svo3KFh-}Ie={P5YxQM}cpe9?J2{z5tEv=FXnSO5EY ztJ}-+C%CfPTEp z*!vf5pWA{N+3saqYkWGEHnIj2zZ%G{S2ngi7_yq0k8=d$b;j-&A*&b@_46dk+Jq^K z`U8ISpkQJ_|3Ak$L5V3$W%o`T3iw^lI^J>n3Lg)gW**3!xrj%=sg0D~j(s*U@RF3>aSn1rpF(JZ2tvIiW1czW4#JR0Z_IC2<)o5DrWy|{; z^A32JLY(e|Y=wqS%EmF>3#1~!v4~syu|7f&9v6uJqEd~1wJi14>?$+pec9f3TAE(Bf^Qgny7j=k$?Z+J#x^$~#QS zDbA4az>Es4KfOu`sqs(Frf;)$w4T)%BP?kjd9t821lhGw3&+bKJTF-l+0U#VpJs`I zs8YQ+n^1iPy#u&FDXRauVwgT+{HO z**i0sW@WNvFUfh$1=R)y#$oX*Jw+QXO7{=X><*|}Zxgg;y=D(Yw7QL!)9x2$0&Lxq z+_ommjHEao-O0G2G>>92@4^@VOP(LDnxcX$)59v|-lVZ-moC53YI}TIkj&d`G$DGN zYF);5xHg*GKAZv3QRSrV5YnLRu`&5m_;UVDOjC?+>LIsz?zo=p-gs{RMR0d^^K;^~ zeYB=ST5@QtefAjFdaatfGfpm56{hv=o{hQPMi(GhN@s1`>;^JkPAP(n_>jiT4Hb6> z8G%Y&!(7nIdHmvbpWPX1Kx)fe^UemP`3lu58TGO`w5nEH=850mE;9$`bazyV8$vnw z(18Qc6?An@c095V(^Z(H;%MB~V^#Bw+_BX%Gog3NN;|bJqH;XcE5n z%wy!Bt}UEeziIx^hrL;@Xp@hi(Jc9|rlM#qpS)68&|_OT(RXo6|G&J4;LItQ6k#Ra zWM{>8BTmLH0J6Z#sA_pVV+Or{=t&7dEIF8+m6)tC-rz=5@8ME9_T2d^b>VIR+S5o7w2mhP`_+gF&0{~^zGQom ziLSpM`uYO+Ltw~#WCPuO^%3T!qxZKf$*0~`Y3bSnCJPzY#jh%mHTj1WUhR+}7Go1I zFUGVzi@Xy=g+1PM-bS$0RVClFMU1j|PlDP~0K+35iUYQnHdjpsVj4`F1s~ z;q3dHu1i72k|in##qD{vk1BSweeoLaQrNt;5p^C|&afMeJr}BI_>e^edCivC?>T5dgja#^9I)r^z{-RVA-SD_fiOo#Y z>LR<&(B)}^;K#0w41U&MF`PJ7@r2Q!foE3G6nIhrV00|`NYC4AuK^Hul)Z(dx37w2 zC^rTaw$ce#M2br=7;;qrZ=%A&Xn~kD44D*_SmSmdbW+iU>Q{9+9hIYy>jT>Im6`z?ZDD;jJOjMotMmx$` z=6N|ZE^euR3fKhns2s1x#2vqx7*}{|`1FE;d_)BIgj!F7An(MYfD-}dodc10b03{O zP>lNFGQN-4!1L>$ovuSYza&Lnk>PvZ+GqY)wx?@SQz8e*UdsDV!9<)l4)$VnoPcFs zYT@d3I$Yf|IN~Bv;xSJc%oY83PAgxos$mUUHMYxzrsJBjOL0C%3sZP>uPf4knzr$} z=3<3x{a_folG2M-tGr1Z(I5SGwT5?SZD*9t{;3DIzB>vY%dLf+9ve?Bv1l?W#WDIk zJR37<;F#sVhH(Lz znK0r!q_1O%5kFp~uGkX-R=v1mh4$8hWSaYaO19J(66513VD^ml>D*kZa?AhyagVnSh#@@Q z<$E;~dSBw*j)NP;mP2D5<$pi3H$&X-6W&7DC~y^1VFcaVv~<1fEOxmpM)0di)LEf!4fpUmr- z$S=KjUHQSQGp7D&$WLjXhP;bJmz)F6Loyon*-09muSwKzC}|SyUY8=kuR35t4PVeV z)0X+!lM&~)??os>@vlWOD5Rmx?WFNsFNLReD@lk;a!Qa<47);e+~}f}u3q-}Ixe}c z^NJv~NpCmx$))Wz^SKL^O6&Ux3g`Je6W%qe!?_xt^!qi;WPP@9_>PA#!{x=3a@`%LX)nHv_04g<3+G!ZGa!xGcjx23lS6SI zB>fM(#bSm*Fl^v$YzBIGyL{?>75g`gR)cLaP#|$9Zt;p_%CRK@!;jwVGwaFUv=3V~ ztV9}cC=ekaef-VRk;kyv9{1(YDp$0{(nk3<(OJi!zAe3*Q-+%JRhRg|I*tJE_xda* zZ`de}pA&gLYUzh%%{4b)NLk+S{nWeQd-?iTLK;BhT^nt&P0c_r0cY6WFC%4AgA6o0G zGSfAUa13&_Uj0w!Vb{%wWe;_M7HV9*z9|rAkZN){koC3GO7(R6JDIa=^U=}Df0nn% z8%z8!Pnyf)`~f&n$&VbNlwoEl>*T z>tu!v+Y1_cL24$lr{HlSH}L zYcZwoneF#Y`;!aOykN!6(MQXzZ9=xja4Q4qb8nV$@c9)GrwTItEm{t42n+riswT4= z2vIF~QxVgP;H8Z32IR52w(!}APwGbZp`b4ruCqHqa$?g;?6gvlPPk1N)3@j(;X|3@ zgwj8&tb!s=29?Nh>Htib>T3Ho0MkTRj_1wf=X3WzuYNk%H>68fRM6OUA0TL;kBl=G z8Tc4quKUVvb_bCR2wVsga z1QjXH=yqJ)%u<-!6JF7qh1(njj6;C(!*Sm(4!P?ipFq5?`p`fio2=5jj;lL5>tdx6 zws{tOVesyHu86d}C4>UqY>mxHeXRgXqto^lqq)f!pY*z42;X(gBj0l-s}k7%>bFc& z%OMwT+TX)oHPh$CsGVF(no}olRoIQAA3#VkfUbvO_{Qq{AVZXzYVz0X2WDZENGntj zUS9ml>Y1aI(sTcip6}ajj0>&+SIBh%L-Brd^+<2kbi(V7;nk_-31+q%b~8>zTU1}U zw(vW1&5^|dbaIm(EjI=YYL4bONS{?}d49<}1f4yRHw#oXyUy9a(6CD``mC)MebDte zXj=+veZ-G8$u>Hj#Ypk{4m5bnKlvos_bd2wUqW>~yZbJ*r=AJ=J24k>cwmQfwKK9I zL~mOyVm_bniNl6$4J#xifiYCODbaOG_Al7Errhc+Bv zvMMZd@ZWs@C~v&4{>8AWs-k-XjSz-!oc&iqB6LmFl0SukUy}RYfiq9e=3RxkB`yqq zaOA9u5;kZpVx;!@f!EMTVT@}fH@7Qn*#-I1p6iK*2l{!u-1krIP3Y5=(eCtv-# z5gj_oH@O^ECA4lTyy`fg3Mfs~H6YY4v?o5y5PtQ0oea;m>-M!@XY^Xc8692c$;|lY zGkCvnd=1@8?P93u)Hu}iJ1gyht0z7jV?93 zAnWJyFdF&Bh{`Q5yUWf2%#WTHW#qWdJq+l|ul-VTQZWcdXp>0>K$Gk9H3@0|E=qki z7D#mr^j!(Wxr%cjh0m7mT^*Whm!qa}+7MgzQwR#=U~+nz?-Qiv)ZtS!7)D^sF@j8z zwYBNz4DgVW8un06h016q=)i~_+D+bK;B((MBj4P3o5cG7hz##bcYJvx61es>mETq0 zF6c%jbxCuhEnv|e@C)^EL*1BLxy{OIWNnoyk$?I|6`d9JB-g<3vNwN1tAxs zkzc7_-7t~D@@;i=-vi|2qlnJVpz$}ob-^&ya@?*D5fomL-DE4Js_#uWqOtACmOSl7 z?WWEo01`wTdVIj97J=~zlt1a=(SGXyYFxWUR=5J=Fg$QIxN&Z0qP|A0_<&V&y=Gc% zf$l_KZIg2C*2kTZ8IJ0aT(+wv7or?r0Xo|DwjgZdPaYe0u}E{hnf%@SPT1UGg>J$Q zH#lP3%dzJRVTUiMWzPEFM)*giAu8&74h(IU{sIg09E>E49lwa%Bv^YZ{a?crWTI5C zCID*s&;`{CtoKlIy~R-e4+g&H-RQ$VDB~o)^q%E^m>;zw^$3rhckZj4tMo3*f^*6g^(i$#w2^F{s_9KeaexJn|r&g+>e-!?7d%mU~A zlQ39HIjCz6zWew$ZuBa4nB67xjS{_s02RQzX?^?)Lf78oEW+QH zG-pf1Gt1Q9ct(I|1w2VS5oca6Zm-qUb{W^cgu6cMH6=ShVz5$3Q`-Ss)Syxrg!f=5 zC0(x9J7>Zl*La}NaT!$N0)z-FJmSHVPqV^{v3K2Hv4`jLd^StiCQ1_5;gukjY3Po< zmI9FWv=D?4{ZsUKR;LJ16XAb=?;i{#ZWcO|xA=-I`dlr3o&@Ihh=ZRF?3PY1bvmOQ zLunYN`D@xlopQXg5cZCa`S8Z1897dIhx^1~%0Gq<9M-gMW^}0mZRrKozaWI~2md{j zJ=2i0a^HTDhTnMBGSEf^4Oiq`b;*Wh{ zMrRO?N3`u=c1x<*p`J#e^y<@z?TcS=MO~odE!u8Is&!FloYz#BjCE&~OLGq9u;wY? z2&zF9WPqOaL5!{L;NAd|4jG?Ubp0bG&yrY?5;Zdv^kVekOOX8 zjEA)bqy|(II5|2z4X2QF#!Z*VT=Ae_{~0Z#A%Ds ze4q;m*28akzeV_Hm7V;WzZ(Pl{AEy2fSxs8`y`hmx9S;1_9+yd{o+kD)JlS0p44ld z6{0}+d32b{^JayL+XLV$7-vmt^kUebhLUI#fuq}jg8xOGL#q4W9U160I_K2LKVKbE zhkhSq6$WX}sLExr)nvh|p9S~!tnLnw>`ohbxDjakRHz^*+xX$MKj`&|4Z`3I6urHr zmx6)a1&dmH>=e&9%r`|D-RHDBGL8n0Y{79W*TTY3FT-1!1vouq@7^aMt99YmH9H~0 zR5$=1nr>o_DS-Bq4ZBi6KdMX96(D85Zrev*d@kn&adASbv>dG3qzZIm_T~%H-Hg@( zAstKpg%1x?yVLkwlz@TE5r4hy=zETq^NTZ{0;r)WYRPtESrzapT;IQ$haX1zCjeKl z6dIZXb8b7^F}OniRY4%!Y~v`D_hdWznf$RHQX~HL8BdziLDt!qv0L*hl#hL?^^=v` zID%;RN`;L+ujfX*a%*ztH3+P1Wb(TBQcK1YZD2E6)E!liWnFZYl2lD{biUSGh zQB@-~a~p#8x(@K<|MX%QjXTUF69r4QmrS=8Q*p|mN9LUDTd0@hPA}(^J`_h; zKl*w2_~^Y_YKVSmI`35>PB$+HIuguB+1Gc*V*j!6gfC$Dcq&Rh#qJ@nnk6at+v2(LeGILX0OluIn(<>Au--l?zBMe3 z@Sr*<(BAY*xxA2T#O;@v$zOTv5vT$DcLREA1BZI;PwbW$Fb<;nrNOMp+i?E*$s-W* z&|o9P!7l=M*2;Lw3l7xLFLHSjcFR$)r}WfMBO7LQcAW-cZX#Ksc*?x(GT48th(oxp z#sd;ss|Doop2AOE&lOX)Ni#RA-=6dRhtC63B?D{JBmvuhid2F32%(E+85%HI)6fYV z(DtJ03ekDsiQV{hpBWP41XhAQS~bzG44x*gC8hncnq%{`4%2~5>3gfwd1!r9YO%w}oB8u9AE>BSxT%s1~C2ud8O~DuWRb>k$1d{1R zcjwDx%kas-`8WnuC@k7p&kBTbm&BeTiz$dT*JTHQH9n(iS0eSj$ji-KJPDumwit_( z)G~zE)ZVx;I9}SHylH@X)t;1lfEp)fM6j_epxH1dK*0gsrDA1d(a$D{aguiV{+Rr^ ze#Tr1L-bXx(#dqJ(xyae43YS`q@vBcM^aKU5l4F|W|CRj#4`D5bSy~Rvv74po_3%q zRRR|*!}b%^L_^+rJ-WbIifT=gIe$l<+X2$dP4TrBtb%Yhew8wIh(pC#iGkp!^NreaJWr9urQP&8h){l&8yf3N_i z(wEV)OXg^+dWD`e-BGqp^E@`kl3yI7AUz{OpOTEKRn4vZLXj{6?8Si+c9gPtQsfW< z(-*K^6Xov-hS#fvHR?7yv`9CbYq-yQf%}6jlozI8-9D!YTR@X`x#OZic(=UBPW+rR z&;8z8T-g*4`bm=Dt4!I3c)eQ!c1-Il&HdRIa_9l0~fElg79QnMFM0U!N0y5J0$mRk(&Re!Mq~APLI#kaNRRZrGy5vm0l&5Z1=Klw?s~O zRH7poMM51v{N1c>hJi?%o-#tv0i%dl5og{o2x)AdDtKXGZ2hf(JnrmkA+{3nBCkwBw!Jk58_M2?M*^a=0;JEnPQ2cEAjCkyGto)TX%`H;$klSO+M%V?M3^b!b zdR<(!gJ5n*LNsae(79bA?;Ny4jGdX#JpJT4MLtBgcXY?{nnm5OTaLNB+jXe#HtxNA z2=8J+^le3E`}OtL?@xx{bAoTaBYLezrWmGu+_$@CnerCegVKWcx5n1(wvR5;@5u4k z`gV2qo+aU{YL?L+Op1CPKl_cg-NPoY3fjCIt~UD_!Lz$G*if&mxv6UM0X>;xuu)6? zdjHdG1&9xyM6lAmc4eOPz9Ouk@o46oJ4Y1c7zyf6A3ySp`0NJ9#_S8^tH>^)tWdCX zDm}l&J}|@EoLWfsZM^2PdwJRewb4iDUS3tq9Kvu4v8L(@;&etw?$2$klNZvD*;|B) zbMe@dk|ur{@k7W3z9;^9+4^`+$e4euot9bJU~7=@Ils?a^6+C45zn6SdNCmE!Qu2R zj<*@#;-ZdNaGYwUPSstx$2-=PI}rgKRm-4vC2s%NegZdIpSQ5H!;D^fbbcDRdb{TB z()F8mm=V8QV~~I^$@nORf}zQl{0gM3kP?*6oqpvkY@7I2zRc3twtR`(Ro>id z#B^3$50z%Dybs4Ckedb$TS>oJS2esU!x>_)+2D zo8BpQHQYB7oZ~XgDHY~`yYZ`l(FsA?8yO}m#>wO(CuUd<&f64O6e5b}U1(;jE}gbR zygEfK@pI97#H6bRtvaad>f|e2#0>XSQ$YO0=gxb=u zq;R%dFUbq7P(blpQHe#BiM`e-6sRrr3LI#Ch}u8$rjD%tq7}nnk8Rj2C7HO!mk?4y zYi}GI7RBGdC+#4gsA`_rHF{&DBWdny!dyway10X_lT^Z)ka}~2tmKhLtOLUxULrr< z&;*?pyqvrDTCD-~ge9MUfq$ktsLT}_46|taShZ$@q!n(qXW_@=iL#Y>pQRST^ahIFNY(>nraCW)dX_wQFczY$dl>uzl>MDwU2M3#Ij09MFoG0uW zO*D?B5>8n%J}S})_A!2mdpUk|8+dFo&%Kd#@p4aAJ~7Cg?(3gWcln`5?Q213c?E`T zZ+-{&i3-=b#t@u|aW)h?;TowG38u17pMMx1lQ-H0{Itd%c~|Etf11X81PRMc{fiJ8 z!?k)pRAP$rHrB#zzg0cW*|9EOZ9Q?@z_~z_ce{PkEFlGrbPdjqn`>F5D0{AKLH8T? zv5-_QYt&TbZO)ptcHb(|l%n&@ke-|>=12r9D>|h7yc_w1#i4TBw>d6qQ?sZ?ByT5Y zMVYpN#J*4wF@n2~P|uiS*`5{q4vj=`TWh$)#}iCqf=Q#MeT9D(kABaRUz8G}dItx_ z$B6HF4#=cgbfvI7v)UiIV$tN(%{O5)sqQ*hcx;j@uP?R4BQM~f5xoDqNuP^jZNODU zaXJ3A&@6h#VZPEd!EmN~YSulwDzHlYt!p?$EwKIej_HrGz-X4o-ZS|>6Qvsr`z=4j zx#XWfdYOTaXkjX~BO1#9G7PeG{gmHp>2fwKK>IC1Y_07IE|CiPQ9!yjiP3S}j-TW}2gjG^PC@(W9r?ScU zODY1>m~4LVwaV9ZIVv0Y!K^Qk4*gt-0A zD;ksDQxU5yycWfK@6|#sej51j+ss-pRJz3E!Z<-0d?H~n+4(((UZZI9eFrw8edKHJ z($=I4{EnLw>k+MN-j;z~{)%B8;EADN)#VhW<(0hS4+ZgsuZbdx=u-j-$IL@syC!54 zt-n8{^_a>p`CpX1dpy%^{6D^m3f+_rD2KW`P|2ygoGrOK-4%+E(<({jSToCEn?&k% z>L8NC=p;F%Vr(|zuH`rtM$CvA8=K6`%(m|}-G}ba=kE9Y{{HlM%=@~o*Y&zyr|0X? zSdS=N^rduKwFX2H3RtHi280jkJWCP+ej&k~1;a7Evx*K3ifr z!lVD<&UXi*a&1?(n58%6R zg+rqQTRsJ5&O3>aqRYq67jF3IvOLNak;uW@JN#7d?c``vS~ZuQc*t7W1GF93Fp?*D zKDO5T`T%?R)K;~_ub(d~5g<+k$5GgfXVgz5syi}MM={E;`Ki8O?V!yxaqFPTbVeZt zIr8(3*eTIF-0D>N<%8ai+Bxso;sM41TcJqA#96EXG{NS}9=%ZjbwNZCtMxOG@8pl zXm=FEeZDL|g;tQe`aSo#rmK-9qcKf>FN+)+Jd{K_`X=)3K`9pzfU+khgo(}*7K11Yt5YI(6`W|wN z0E&lY57pI!k&t#9rrNnuG8Pjmh$bzcy0c)UD#a^oS1F8Lf)~9>MLgzw@iwIB&@j6$ zWEWT#rL)OyOC%=FFasp59DP1uQaI9?Imvt%Gs$2FB3|p@&2l{t@Uoo9FtA-JAYxqZ z9Cl?IecB2I)UDIS@!UB_Bhw9E=H}X2O2>42J(iSs+zYv6k)#rLmCeAuiYh3`eiIMA z=C8q)!x{%z+o$>9CiW?yxPxw$__1xUnw-gUW%*uaZ^3$ssqe{7bwT3WpuH0{qV6j8 zQGYEhHUzd6o7V3~L!93|;6wD^^W&Rx=rsCSKPd9Qh)w!%U1wr>bh zeYKPfbyKsjhtl~Z1D+Sd@2S4DcN>}Al^azx<^x*Ky=ux8J^-D}mZ5+16Psk&v}d4O zvlXx7nJPP)$_Bj`H>X!_e3O+M1tWuk#C5q8+xDEK>9$G~!uOGca>bOK&omTTYcg}6|9_X%v#K@T3sIuLd=)NCNbUYDJ$G z&~fhX@&vvY9c|Q5ny>VMRH?c)X@@+19#Nx%cN|caT<;df@kDQ3Ya~b1=bTWdg;*(z zjaARi&y>9ysIHv#=Jx~-Jw0sy*EL~Ikqk5R;msutpUo)wut?)7t=vzY_!#rSko=SEV z2q^N59@R9=D`=FVeob~bK6mc*S|vs9^W2EN)lTg8^}+aP>Q&*aG5c!<;PKFM=w>|_ z(x0BaJk`){1a4@e{ybEh-m>(fikHPyz@BY|jM~>& zn&`onOd}6~E)L3^e9-J+gROv{cvPrPPCP->_m^kHTl_&0XN}1*uC5q;;Em>xx@tt7 z&!UdF0az0}ybD@AP4uHkVX-{JNx_!8MXn18+8kg__WL5GY20`q)KYc6olQ>F@JFTmy(wwzK zp%JIh6zn5jm8zrCpsPMtKgTJZd-9ghL8YX(1;^BlW}yS9B7D6NSQ96o>I}PLu9y`> zqJ=rlFsL-UZNu zb(TN6zw-$TKAMc>M048hhZLyX#*#ZH1R(%!2R`{Jo*Q|hnj#&-%yM;!H75U+PYGqu6+%s$9B5bnA&0avuAEbWPl!bwawv< zb^ZGvA9a(AQXOoC1AD?WO!|J3OF}BiGnz^!2ss?SlQ(AXA0A4;ni?)k9?%fwUD791 zanH%suKc2npV%PHv6yC1&V^TihBsuQx8ZPNq|nUH$B8sOc_sxM7L~WxU~R@M?jc&r zv`Wsk3Q7KVlzoWjKa5MWUyLP-}#pBhW1R1zE>Ncx_98M9HkW8j~-q_p`4QKj^ zyfKnvgxC_HP8d^__F%-Nd>fY>f7j3_r8PA@j^LkiF^!e!DW3%9`k1HM^vfC^J}#vq zyFAGx>S8fP#%zsNBbi}h56aQLc`U!E7+vlj?fNPM&cYDZ75zgE_-QSWL`nu zxYx!?nI*EYiEwklut7jJ6^2yeMz;89&FoC($APISCrA}@0Wjv2+}kF>GfCABdC)M@ zEUdSQJi0D(wyF^l8GgymznP`9@A_Ub!>1fiG31A@0;bsyAMgC-R};4H1)1DiQhVet zt(3IX@c1tu@k?|A4D>LP+$iy9_tRss^-Wj}ywO8tUnCS8#1ndJZ@(vX7$Hi!@k&vJ zPT`&TCT=((GUTj7&;M3PyaW@e(tyC-vKaYUoiW;|o8?-7Zb#nh{xK6%|_ z6Wx>~qfdF=gd*Vv1dsY|w8AwsL?^z<+oY45Ydm1$6^ zm6sz)Yw~E9nN}$|<})}Y&&Xca088dk}ITAJkP&^W~7r!)*|M}fQA>dbb*8K5Oe|S8D4UP|fTWWf>M=qwR zk*b=Lts2|vGvw!l%In*l_9`;DAdhQsqk8J47C|3Q4r;9&Mm19IaR-2njrcgZ`4p-# z;zNXCBceF6sEp2jcWNx0V0@lv$Pm2C;cb0K+_l?Ddn5ENlbwSc+Z!`)XpitBFrd37 zCQD{qwvXsT9q8^MzvOk6xM$-<+n^p2PM43MBa7*Irixiz5u4Zao1-NRtEvYAYEIj2 zM>&^ROarW6KouU_!q$UPv!bKesF8K~K7Hx>hZlCkb+*@-`l8e$VNXeZz!Y;I%)vSc z1eXJzTJ29OpCjf`U7!afn8sPcF!PwBw%PCx?d`)dEU|spy=>T(wSSt#J0F>N!Q3P* zGy-nFUw8Pd%c94>ZAFaxCM)CMKQeP})z0JC^~E#u7N_adr=?9;znnkO#9I6hm9oM& z9HjU6Zw8y1I$12~=Jsm!DK(Xv{)NAsqW$n<3)Wp%xi8}Z1Ow8 zv)&l`+3HUW?<(Sl;SyE1dDOXbb;09IE7!JDnHLduq-$1Ybhm*5BM_MEk#1-aV3%^#4>~TX^A)LP9j{gsPE~s+s5cl5D zXUcA?_{AI(%f$qQ4!8W33gGrb_jJS6`4pOXkK;%i)1R9z#xXusNjuQe=8+@Vzre#6 zoF7I;78cH9UO0d{g61y{Zj!0+zj7Ja+CPetZhA2fE{7cU=BgM%6&vYvsOh#QAQuAv ze{jkjs;FKGm@qTKQj+A5SYU9kbY1qqzgL%7uyIlzUGxgWri)dy*<7?BCF!Qj66}wW z#chX4^F3ohApIXQEMde|fL4EL6w#^BS>|__9@?<77c&QdHmtY>I&&8)Dt3iQmOGDb zgNgVU!IBi{#$_+izRfy)9P+FshN}}0aq?9wXx4=x;3mb@*P+4#(z$bVN>tFKu;EH5 zwg4l=NgnCEW2TY!1J&a>XeGX1Ro52vys(BZ;lg}xMHN(WyQ+WTm03x4a6g^x>rQeS ze#EG5m*mJYo+_j7?)``OK}rEwOfw+@F5nb@xIp>t+DDF`z8QL{7qv7$@kGojp9R@+ zjuqMyYf@q8lVp~m^ssbt5Ff^gc3lY7Aj6@-IXkiG1=zw^dXdz@>7=QFFh==Xs3#?3 z=aFm&1pGPt`U65$yE_Hf-ivIlo-(u&C8Ve=!bVE&RlRc>+wNO(mzqhn-n|>c*M*{% zeahTuL)%x89%>t31adty92Eb0>36=pbnK~Kfygg-I-UzvXr^oRNkeO-w^#g`p4Fd4 z{xyB+#Z61g{N*;`Td9PcQ-@%ckCx3Ywa8lYnHby8$=Y+Ah|b2oCUC_ZUu7IMN7#zpoSp!$)TgFu@;z(#gTT{u%MVjKx zA%Db#>FETQke_KgfJJQDEwlIF&GOqm$lyGQyqw?aQa%#%DB*8R&z!Xf@$dv;RnD`Y^IJ z+?ZiLyp*3^Zvyf-+Ib&~?1T{lZ|?629xZAV4}uaQ+QkKANHy!m{UsV3WJ19cE&1kf zWjWx~%kF_GfS?&U(>U3XO@>PTz2{B1pMpv@vb#X-NceTrhgq_ga zn!StiVaJ@NEq0dA;2BL<{TAYn9Inlrt!!d}PLk2`%+dVC3^JNjZG;dEoOxSbVVpr8 zb5Z7$tVq@I=&UY@(4=ZNMQBjEi8l9?l^>_(;hSOa%b*9n-OgaAfDg!-CZH$F(=oR`b?Qn%Pk-U-#0euQbD}t@ghw=Z5ok zD%c~vG~@d+U*0`W7qCkG%qJgAT7@SUoMv;7giCrDLkis*#xB=d8L;hI1K}!RWqZvh zqJZH)7TFq9)fDYmXKPpbBnryw3NOVAw!_w~qgV{>&fghjy~R%JC|JmgW)j8@AqGt+ z9TLy-_*ULkXl0c4bsHrik;5~u$ib!U7|f#Yfa=0+==@0pdk@>4}hURo?6 z2BC_mZ}WVur-QM%R<(?lPgLbb>gnM!`w^iq!k{A%Gh2bAg%vKl%S4igkHah8`tTa8 z4&aC_7^YQg`>jjXOEOLJt#pK@2(?q$@3WNlv&Dam(UXW&ut#`h zwW}Tbp!pU)1(8A}_RW@@K=FDabk4QTRg?$NY`ldJhP|8J4d`utd79)-X5a`M-;TCL zem#gw2C@tiF!lW%lDc$AJAZZ<9#`OHaX7R{mqJVa}eBo}N8mIVujf4$T9ggQ8 zFY|OzkmAtLrd!lr0iif~&IzhiEO_ZT+^Jrk_K1J-{p?49nUC7CWySRb4VqVc+Qt-E z4U&PkWDr{iRgzoyp~6rj(g%u4$_UmB^XtRq1O>IHucv=vQTB@al2Co_gJ#o~cJPsinp}jgtgmczQqHQs zte9O2WYMx9JB#4VL%i z0n+cM(Sr5&e84ZwJyGn)YX`253WI%-_>NQ?=6^(WY^zR<((ITE2>NZrdw6Be;#!ow}f}QzZNAq`DLJb`=IRI$B_W%t0q7PckS1dGf35 zpq7l;iVJ{iUKJNHH~I8@AqfMmsTYXXH-p8?F48&*a+smW?u#w1?1Wp$oxo=bzd#pt z#?`a8VWv~f-(@Yo(pT7dWy&iG^zS1NKViImpWhrTGSJUZoij}Leb$5`_D#1863F-0 zr3NSQwuU#YtQx~FH)<;Qta}>X$8yfzFw$XJh2Cz zuLNO+v%ikTtU38EsA`hOfS1SEI9;1$n1wanrn(?=1LWgaA#knX(Y;$3gZf%v6pFFW z0_>5)rBPbJ>YfOflyc!src>Gc;uPJxr8?;)4(~W1qWD{T0Up`u_))n85iX_?Bpp%*N`!*mgd;@ELf|AyT%du zdmthcYZ64G-m$6zD77!TX6>`l{I6>k{= zcJ5_s!82~##|QLeuK){Eke;Ld;K0#ttX_I)IncfHICuDVeeY+}hi@#U1 z8>PmbOD$n8ObiXIxZ?U1KpP0kCQO!b3i%pOBOcfn-d#y+e9&QmeO=OjKU9gLaf;xYUM; zDy0&tb1a%^v(4?L2$o--9lrAaXq3dV1#?dT*7BQ;p4dN zN_WA_s4#bcndo5y_o;an8&)<~JPz)0jlba0GZ(SWJWMx(6dqc21jS38jVa3$2GXlb zds{1FMq~mvJW9g=b`f*tH%C&A(=;*I4xT(4KHU0&*}MAUNXc1-|Am$^w)iQ@+@%WL z;X{8wuNZdaN0n&bjS!D{q|MfS0)eQ6+ae1p6<#-KoHkqCfgLXs$IukiW*ZejIg{{j z`Cl9kY|=oXMifHk{96%2Q-J;SDNq7{yFc^gvbgt0RBhuybcH%HmcCN#gkQlqx0oSX zD2H&2&~tQOqSLtNkcPoF%h{wIky@*sr4xenSv@pdPN{OwvbX_q%%oG@TZDSS*!3<& zZ9$U%_M%JRSlYZv0b3CGTNerO;caSel=dSf>`%n&dHs!!Gwk+l*MfXfyf8s5<;R5E z2KSv79^(7u8RnX#jam6L|wBo9dZXw9Ag>9F6U_l8@=D*R#BO5)7%8%Lrd#-h`i4G0J zJXEf77}vwFM7ssb^7)=4pVfx2FDb7IIhqxJ#O0zyY5ceM__M~0x~9&E3!POPg`|4O}@|y|1Ux7-ycIV#*{Xx$|&Uo6|9*i75I_ zAuSO=pV7H$F$^ z%8~!btDX7?#vVETM)EGX%NJF680pawX%u(|#i_)H?{iP>_z*K~?noN;lBw)PpB)txmylDH9BPT!g^B6#y_=r zUL1S)9#E?{ys@j|%K0FCm?KFufn==JgYDX(o(i3o5ezD*yOGEDGd0e#;XjNnvoz%I zXv(6%1m3UH{B9=fe`gcydl17x+g5SS{A%`q4(g|T{_zct#(c%8l1NVl1%;tmHPhdF z(Ow(%=xR$FZt_hIlGq9{AXpDXnw;;*xgb7*UYL$~c0Fc?U6b{iAU6{|=$a$kNW;cc zEnPl|L>M^R${QN)Fx4)-`t?pMfSIN}(M&J1@LPW&Xrz$KWP1K$v`Y_K0r2PJNLT1I z50#ud*Go7tg(9U@6RrXax8mDNn~T!uY|%93@jEp^+_Z|9xwS_O4@dxx6g^1}$83cq zgJvD5LOTJRj-&mCEX#;E7Q$w-38#;GG2;!~Y{JkU1|nl*Xjc+I*8ox^y5m2lmWAj3 z4C=K)hX3)Shdab#xUEnrT$Xt8Z%>L zB}BG$h1D$k0=w*p9$DDBYg%`9L@n8TCkPtq4AzVnIa~aA#hPyKJ~`MK`-Eo z&NCwKBtYDKZX0|33}E8!F(qcInr@crT0^@t{&BqFl0lT(Lx7<6wpxh+h#QgY80^u= z^-cb$mZsfflhhK(^xT7E0Jw`;T;JtC<}gL1@I{@b+6?*pVnN_}&=;XPltt=I<@+Fi z1r#Z(xsy$~(EIoIpcQkCGrAUH%32v*7$FZbR0d<~De%Fu?v~T3MTs}A7V4qPL~aMz z@%r)Jwo8Rg0>^{Tl!IV!NA$(`28&;0$IxcaUScy_W-T^Hy|VmS4|e?qi<8GUxrfc@ z^w}K!x)7h#T6O+O)ZFEhPYrzh#|$>;)mZ*q-4T%$)k(&4&V|36WAK0+gjmM8|vz<$dV>{&83BV;C2`=kq#QErGf-^krv>I#1;>7 z=QThLZ;rVa1K>jmE=|T6uYx+79G6IC4}ns(=X9%8)cF8BWp5n^Zg6V{wXCd)d;CA> z^3V`Oq!QmVWXQ$&IED^MGU?uV=+8Hw?Y{8|MD-6{LN?j+_pk{yju)TNJ8 zDOAObbS~Ag>oIB1<6y04n4sSd7AkM-@Ht-7n&&8VymIa^K)`mIcCOfY`k}K;y4^rI zsm2EpQXG{C;7E+$uv}eZD^Mm?GIuEwxJv*lz2s80BRj&@T?{YIjjW<^l|0gWK9X*I zZ8OnYnHt>EGYpbCukcAn{SA!x3kM@GXY@f@Q?S+pcsrD88ehzWhD)D%MtQ<+xTiF< zD=m4)>3t2*Q1n>qG5+9L`p!}!`lt`Du~X#Q3M1PD^KB;EZ07zqDHBwa1?JVBqwDF4 zO-v3%7L^Yj{g*QeC~47_d6!wd{Gvvvp5<+B1b#a=Vxc8DHlPW6bbYSjQbz+tXihk-b6YFu=-PF`vEB*;`+!IZm%pi=wbQW22aMvhb>w#I!Qbq_iG~u z%b+NG^}onm?sl=*hZgCMT$g?M>H*Ur04k>(_CL}%Vc$vZWWMIyaF2&qtO0HQ%ScAY zG;D&2)A&KOSUgA29E?C2O3_pDEv%Z2#b05pl2x)O9n=A~Qx7|i8kXlrJ<6(YhZ9{Y zE#bGU-n9+BHpmKy#IgG~T1}U}4Io_4GQz&-cs|b2!m$6RSb?2c$ zx~5jH8;GjLrw}eRs1LDe@UzPr*OT4WZ%z3?q!+lHdZtOwOU~j?GRh{s!`MS5OHu7! z13Yff5q7RyRbnwhUy8l-e(;<|V}jn~eMCQ=ugzr6dPkD_tTAkO2XbH!$`s5SZM=L= zlU?h%eS!#PH*Bx&NVQ6q z-p`mELWw}Sy=azuapBac1*YKp7lQw_)+T3?^v>*Fv5KUr>rdS1UkQz?F&_{YnM?$C>xx5D&?|raXjIwBQ4dwgfHr1HM_?r zg)Ooz+KSmOl7I#@^P0LncP>k4N1rYW7FGC%>X#BOPkz%j=qU(1+QAU8@;s=d>& zWpHZTt~q?n#M!3(9x)i*MdaXD&a|m_su?d!f-P)!5(p>Zb|MhLnDbga#mN+Kcdtne zMuc(;^OVr}oSNw!c66IIr*Spe1ciXHzm@n78Qc*lH2>yWdOpQ+1kmSnN{I|rGQ9LJ zn_uzd@K88)MH%7@7*ydJ#?pQsYudO{;AJ|&iS%-AK?uqx;&j<5 z2XQZ-dGWmUsHmtj8_IK|O)ATe=NGN?VSmk{OtkZNaba|(C7aU4BzVHyeeROg-36?Y zA}{hxCp@&YV3)*B&GLwN-OqX|Awu3%6^6_XoO1N_NUy4G5A;DTc^XhUDdLA!v3Ju0 znT=fJ8~Wv(cGL9pEl%$$m`+S3A9zL;@8edV$|1=gZG)+$mQ5W3Qb&{36sg=G=NY%_ zRUy8KX+GRc^$^`+_+;ZtO?JmW*je)ck4+;t&rsDChGz4~1}@s`GNS^|(=unnEvaZ= zy#LD|d-%;sW#3VrEPfHi(Y|FQ-uyi{IdaGL@RN6Vzhs1!>u=Ua|H;6{zx`hh??aF& zajX1TRn*^B`cF+dq$YuZ@Ap>@lE26|A1pCrevt1E@m6T$&EJpP@ zw=kBKlp(9;J(BsKhmRf5xGW3{_KW!bY{R2B7q2a@3-xoPX?JsCL?Os3cxG zMUk}9MN6RVbTLO~otmuJ*6hr`Qt3Q|DA2P-B*!mhujcBh zlK>6`$-EV5A+#SRIDjvJ)*K!umN2j{GtHRk0!yzU4@~+eM4uIC_tE^}D%<);$$JOE z|1MYVeNIKj2RBl_$hc}hW4gL3Qc%IinZrySK$;^wwBO=~nGGOce{<|2918$2dmVEQ zE)fpXM}V?{(=OCbJI*i8YbnqA-`SF&7E1Z-YB1=o@`x<*C%fYc>%W;B zfWtA~G&-0>o6q>`y#$3qAONqw}|3G0ENI)4hZ_2qd^OYIAXUB32F3u!ADz= z=1tkRX4EPeY^6dr!$VfrBny80Rapk+5p>BsRSWgEnlh=!mtT|e&mD}hI!sz*rGBr!`q+T_KXODys+{B>?CV=*!O4%`AHwP<8OSXYf}n=x zuY`M0H-8#5Tsr)y?%ORmkcFW+rh6d}<`39|HthC zedvEkFcW$^7?JKk4c{mS36YrpfRPs3hIdZOiu6E40%mFfHj~X$8qk|Z=)3y9C+hZ) zJ-@@ja|S=uVtxwmTe<*}72L|5FVG>!d|xUGUYFv>eSHhIwhsKqBSy^k^Y~xH-@ZJR znf1}y1_&w|0|+Ycd3}ffKA){tDrTN@UR0POxqTINlR`XbSml=X-!~NVjdODL{s~V4 z^(Es?jsJeVigJwb-~Lwiqx#Z`@+gYztg;y$|4nR%9blB+K2nF5jTbX%Ii&W&e{=on zrH!vv2Xp#}fgq;?0a`AFSp7GbfWS$>p?`1I=EuKmDFn%Dnc@MFgGCcGY6pEM;3(b? zY3!HqZyF!b{ZElvlnaL8ekp`u&N+Ae{ymZJ@BeQyfIFeoBq@4)v$_5fk?LFi`|Eog za}H1Z4QIAas=%5AU}EZ?n&Ex%Kv<`Ss78l_-cK2zR2>9OThKxV6}mE&srpSkDnEWY zFWS$;zW{CaKYk4VCfAvF<^?0dXxBVfV=w>LPvsa@zk*nS)w76sqXuvVw+KQaF> zCpOO?A*2UPj@Z8#a?0Xr4|_t$6zNcTU!-THY)t)oPaRpf6?7v7+q9O6c=VyagiA;! zP<;@4-H=Evck%~$M(JHyQ9Wow0}lU^AO07l{P!I|dXqhkGS|1MiR$M^$>*=A1f9(Z zuM`_$y8kQ!CYc>l6S^$lvOxog7YMGZW;@8H39$lOYs24g21BE!zW6SNrsTxC`4Q54bQ18FpG3?$4_81y=?JH^SXt-4??XQcf>#y+A0H$AP~<~&#}*V zxbyI|0!4Sdv{EGm;GnV(SQQUG+t0YH&=^VAxi$$*yF>t-0xsL<7VVL`=P0n3?3X>B ze^k+dP1Quif`DyrW+wgExzz^0gYMe150ca~fc3>iA>(Cp6(B)(a;Z9F!2gl2bjDws9@o!qlMSi>i-r_0-bfYX0N$w~C3v$E66T3^5pXH{a!kI)%MT|2 zTE2T%FvZ3nw#SBk_y&m&N}U14+w)j3w2)6pY(x^su3lm&nNeR z8PKI%whi*-B}cWMmU>_g2$2}dNUt_BMfSh_=K%448O?1UiwH+vsX2QCG_fkNHPCC( z#B-I8>cp974|glPD&19Dl5Pv`4{UW4Oh>RPwZ;Zz8_tXia%%No`i&whS+&tCuAt>{tLO3W92DNA z^M>+SM=sW=LHd36PMXf3)46=wfK)O=he_4;H_#WJo|oArcWNP;hPuh_xGk+$8St?8 za**iqO=ceuNe-qn<^wZh!5|T?UG2_it~_MMjs5Vzn=bY};oJc(-xVgc-4!f#dKB6t ziYk6lZ8fFJ(FfQ!#*xGSVihhR91h|(+N_O@q@~E*+qK(VjFJerM!xY-R$KG zcw$x6%uZgXi{FQ>UV}VBcw#ZhX2W_&H2r+s(9R{lm{5yt!m_JGz%r3;s~J`<7_vIR zR>oINn5o&x_5*eVWQaZ!xQQ=UYd0$B$-de3d+e$C%*#9B7q>6Hbkl1vzWoi_AH0SH z6RGGd8i~BZtz_^Exv>vQt(z01uje85hQ67RqcPpp(c+!B zh9>cm*i7w`HvzU3IQS~ta5rz|O}K>z6*L%s!~#w^wL{@meorJP%$dtQ%3Ukv^vsJZ zlTzY!V^Fyc2|}UWOsC)1#0>@4*Q4JgOcE(pP?+=#W=6+7dpPed;z`G<_cR#?LAqXR z&e=QS3Qb{JlJrT4tB~lde1);+;>rCho}xNuV-C?8fLd@nBjsFzjdLcB@>=&hu{aYp zZ)W|8&z!)Esq{tojgZf2yI!{4G|2L4r$y2&!UBo8<&h7i^Js+vNNaaE2C-xqm*2q-zg^u0DSIYP;TK8UC{YTsIB_B?cqJn0XH!)-H0Se`& zg@1{+j3SN^oIXsQnj-{%LfD9I;C4%ku1!|$sr@Vu@gy{h+{qgeS4lhgo|OTy?VrhQ zdN=bB1zhEG-Q>$TvTaA}enR%Mx-P#^O1|5=ZM0X|U^~>ynvqDVz!H^SqiyL%PIY4n zMOm+P0B>M(_wJp>i+0u7s`vyXErOJUduP0W)2%)%rKL5aL>BLGF_Rq9AGfk}H1V%8 zYUk}Bho6@`U8B@!c}rQh;pTHvP;QXF5@uwvKd;%8zL9tmBIZyEJ_Pl7P-)^a;-CpUW zynYeH6W7ds2~86}YnIL$+|P0!HNlb(>TRGgTh^uY=&k`wQ*vxL3k84Qtd-Oj_?#Ly z2?kqcy~gsdoVJ(Ra0atOX#=w6`_XXoCw8Y{xE1}sK&2g2Xv3Q?o!RSpb5=q6U!H|+ zUGO;e#mY5qXLQEw^d!E+NyG8vCJ~6%0@37Ym0`JT7vViLiuU%NX?4nRn9=MDn^b!6 zF8GR6=6!yEo)K1j@)6KekXqI(-j$+sL6fxe&Wj>a&jx6iM-{Mefo)W=`6|EplWIjP zE&B59K~=TQe)jjthjkr!wQmLSFj>@`3gWmA3H6hwwdXFM&wgg8w>p*ixaPp3tKrCx zGIe|kVSWmIF=XSC6G@x&4VT;yDX+K8lZQ;iijIK->YX}GB*$U@7Q%w;wyh+kw@S;3 z3Y8mmm!EZpL?2E!XhEr+!tc~S25a{yRjXR+=9mK)2R2GV7xFl1vLOz&7iTS+j_mOI z>9iLLiEay&hn(_KNZe$Vy$2DqA)jL@b_D8Eru&C3y@GCDK5>>erZye8AliTOir*ub zSDEou`GYrp?Yw63xYslNdBRl@T6vh`qvCbmzzDSNgVb)D-COC@_~w{dN~p&L?L8#R zY9vt2{l6$AY}%CFA8Ow$pmi|si=U`?eSw{|(Y-;eLY)S7bN|JuxFK4`o#O^_C58{r zi#`3Z@3j=%E=mccLfh>|!P5DweG3|oe|VDuf{jV*v$(H5v(7w#q+NRlS_U$mRd`Qh zJK=9a>U8XAW5ex(KGtJ8n{T8t+sfb~)vZ{E2amy= zXVsr$W*)3K+koPgCA7mg#P7ES=YZ%xTusV8u5e%7+6+rt39|vZTIaBy9!+)S@cRb7 zk09I~In^-8xWt>W{|uW%Pmp;R)l4Cf^iA+|vgObG-nfsw*VFKKk@UB#5KLhZ(IiHn z#I@OAgiZK3KK}qh?u7nizGc&wQdK4ZO0IV|TeZ<#(x=H_m`9*H_?bsoWF68V?iCaAPdgo4NtXaP2>)WV^tMgjZ z8#h>mXgU_E|Cp4 zgw$?sFrM4G;W+%{>{-~w)=NX3lC=FhtoU?%QF=F0fRl}xi|%H51SOxa*u8ew755Y` zmss%Nx8GekI3G}A3?U8LvI(Uhb#6eT(#cJO!(Vr%n|#IdK$R;%gnf=O-j*dU#4}*H&8xV*`=5}23+0%*$* z&utE$tTeyMiSd5=-0s|dIiP$!$IRE)@O)T-Ow?dN+w z84tyMhzV?1(BJ82wj@PMb1|fK^VOSKwXHVwY)}d3Brtxr?ai!=_vd8MgS6cE55tee zFz^6Q`Iu0&@VU-=a(p_Ft8zZBmVCX!tHqyp4p*##U`Du)zlq~(tZPqDn^PC|v;)h7 zs;Ryhl9qPMetLuIOLf-RZs;nh`VCA5d5zqedk7rs+18Oo5G#IlM2HOv#Ob}8w`v=n zG3YGtOf_Z*t}kqG(pl7LwB|mSQ|WI#?+0wO)7sl;pS)jjx5Ru6FyPjtI$p9j$lva< z(8@>mkuJ4Fi>=1msj7o?5_ijNjTtlnmw)mx7ZN7m);|J|vB-Pukr zTr}Ofx@?!zLY@$9@DAB5BU1DMyCpXAOZ!bWEvL`v%M8s$mp_`F&X%L@F|<1_#9Vt(P_f4@%c~0@riKGq&EiidcG%vGy_BZ# z7HS2Ue`r25ZWVu*5`YlPGRt$$D4T3vAFSMR=Ox9p#IM^mv$~Zdpx7uk)`Kr1pUcnETBvXh$k$MtWWe(U}}5OEU6E^;sn(STNKNqPM7}nw8Ak zYj&zvE@U)-47rq3e;)o-RIS2> zSLOK8c5j2!e&q2&IogUB6yLw9ki)R|O!ShU8JeK3Buew=aQyV*rb~&Mx*4ImM1VjP z@j8Je35nkBsxz}nJG#j={wjFe{g)Y=f`^AkucR#(J-$m^$@*FbLUc!SbtwaX7~xDz z!xk)syhv5?N`xu5e2^b9PoMR4m=+A%?C<=P$L|i`Xt1x7QB>-*PU&&@{Wh6JQG2h^ z7TamNVac`rm*oaA4Tq9@CEX*C+UZSa?~aG={Urs7!dq6uwk(&~6zOD;{2!9+u@NX} z{bIJ8RTyr|g78)gZZStYYV1r8R05T__pFu zH=}JK!d~NrkP2Pf3_1Ug4c5Gg0$R({tmYP<%^*jJXk_$SfJE&E6FiQN+Rt6ddme4& zV9YJHc%ws5-r{WanttJN)md7cy%VQXFc6cUzE&m#A)Z%5_v@)hHof+7?zaH`SM8^U zhV}qB<`c*q!8zIgVgDRZG^?v+aClSca^GW*ZmT2jvoxP>LxMk z(F1o{Zm~F1heMyo7l&JU?|e~WY7i5Vp>$||8e?tTHakn@saizC0=kP4rNI*7wttwS zuw_%F864evCtx43)!J{}mG5*^Bv2Fw?x(2AxRWZoS+6{*`7uY|~s6n)3B$@OiiQ zs7BK@_3ePvM?2^UZX^1icbB zl<7YLJ{M%?($Cufa+vWOAjaYJKo|Q~rko_}?rqERIKYMx~)j zz$&MHCHjbK(7Xe`Z7m;FO~tr_%k{h?xA5iogPtdaJtT%;QiR`E#}{qVtASPs5PAtfJ>(m zq*D@DVa;&e%KMzJbtND;DK^*cyRJoQnl`BMmjjaMlng^qO=X&a1)6W)h5HBUUQQS- z>FMk(`KnNB71rx+tOxPDPWGT-x9*XHWF44>K3%*FEaMJcgoqC7Fh_LATf#wSi>f^x zmW#-!938`noNo1p{)(qXz@)Ejkrn8In4vz<4o_JzTZO?fs zDRHmO$K>~+WKrmJL)%)ph52FhqtT9nDt%Vr_9L~5B>M+r;LXt|(wI~tXo!Pv>#sQ~tWLe}gV z?0*~B|Lj?hwq+F#z4oEB^?rzC=}}~n8s29CSzG7OsoEPrOEB&`E)V7Mx|E{33_r_PKq05Q}l4qcZ7mE6@%)y71fdcRvO54_%7e)@nd{ z;`8=I!rajI{ri^apBf$9sIU|m*IiG0cWmQH$CeyfMy^>kJGjQ$JJdT$`(+9$nc0F4 z7YpZvrD#EL)N38-gvPY;n{b;HS{R(8wf2=bG=4uIR7s2sV{^=TGIWZqJFrjJ8WQBf z9-46km|*E;Rr`Kx zqujZAx*@R09N6`7uMMAu)waY1F_$f@eId_bi@yJV!lW%XTM+S)b!|O z%qD2+Q*_pDcs0}sX`OisUFDO{nwTQvF|jnUURGFo_(Dp!eLBJ;QJBW*))40HvJ5Y@ zG&PadU2?1u@c|&1YM{l3b=4yDtFZpnYBG`9uw*6gypRu<(u{Aq&L>3D(@oZn!l&2rjCPOcr6?UuqD}jBhWoeL065>(z-I3 z04sB?wzBKyUbR_K7sOA8*GC4D4X*50L`(PmI$T?#pS~O_JH9p2XCiFA0E6c$lh7P^ zc}-vmAFqj$PqV0@wh#u!oU0U+z%i)_lhfb_H1*bX==c2m_dsp%>pt&^7bQF)Rfnu@ z-V%ug9uK5vuBPi+FRdfM9Ph2> zu4VIBEyYM{9X+v28=) z8eNxgjN|UN?*}eVoj+>;M3o&pTp=-V;M)So18jZNaZdDds@^Ya=JU?=Gsy8%{HCJ@ z)*Ecf@ii1^OL?S^Ya-9+T(_}&`yv*cQEk(2%X9&BOgzXiUo)iNiLsIgN93PW^|I}? zh@bkFTP-sgZ$D#VT!Z6dX`Qfhy|-ZV`rFXb9_9U#%C=6%?1A)~4DBxxQ^CTyszY~w zulcR9`iO*7ZSdVez1!K%O-EEWyw`;yN@R0 zZGUqauKAfvBG(l7?FxL%+n5Gdd*$Ku%=KB-HvoDadAzE?ft~|*toE!M`XMNXP%bTN zUkn6ksBtmRcNWcDthk8xx=(LJ>pCluW9+1b+Gcc1T%{+!r*Gnn`do%ul|kaNuqJN6 zd(c1{qZas3kDZ`AG3v-LA;P`7-hNncc3}e9nGn=)#fwu^cuSK;y$M*$Zwq5sC+}=vC@z`FG zG%6FB9Cc$dDGHI{ur>E&Y8<{sSv zT*q)yR}QORcXcBc`!PS>W6bn!+`28IMR+?f8o1Z_)b8$Pf7!q*B0e{gI{uZOOb&y; zQrZ`KXZI$HoZ#=_^7@ny`u%A9{(;GRk}hmm6Lw>Al@D7lQx_9R4ynH>NC$Zv3{EqB=W*_lION}0iJf|(c5`PY zQ-ku_G&~Ce;fHx>B=@!WhBmj|%jSm5m(qKH_>V*6M|F*XXY-@23a6nCh9p^$iKmFM zYWKf=z)Ee8-3mrdE%Ogp&VP_8kQop?Z{yybL=t{jW9~(xI(RVBD0$YlgW3`Nq?^fIoPE)NWt(`5$8+q!#)p%bXMQvbo$U{Z%}*i1Fp`7bE79QJxui zcY~8U$LE&-C?1U&W0ri@`&Cc>2)P@%&Mt!*DC)Vuk8yxE-K%^~1l$ko?(Z7Oq1J*rl%$%rCHF-^5B9a?g&)VEze?NR z)!x|G{ota8OobIs5;Q&eGW$s*#soj zM;?N)AxXFtPFXta(G5nw^_t$+m)A}CAOJzQpUQ7T^}-I5H9mdwwn!VHed>wi==}?W znLA`fw5|u8OKd0Y+9(BbYXq&$4_ah1o(8TR4aGaNl=cMZmW9Yv$hic!iFTVFUsH+i zYv?qXsUGxRE;Z-`A!fSvZ>>K-2|sxVH?4D2B#%6CK>4OspU2R1;^WipUmuD1UEz*l z2m1VF_Q$!$RiS&`<65p^UHCuD{^syZH5ZiyZP{I+>C%WO%`X^P`^%vYZoNFAhFOSbeb+o9YFMe<$>>JwI_V~+uEBFD%NdMOt1Ixq7emq3nn9WU1 z)wQVwq-&$~7U&w-b}sd&qMvuf_P!d$Yc;bzOSNucUEa&X{XT5IBitb$7#6=+anKg&DXKxuNxhFNDSScypuFd`J+ z$lid+|5U}q_0>4`k$T!h^SUAN-ZozG^q|*Fh)V)OAJzb5TKDMhqk1=g#xjU<1e}zt zvn?U@6i`l`uO^&v8TV@0LgWzfQdDcDFkJwoe1S=KW9H zv0`9T+z$ZK*(uANq6NCX{~T6#KucX3!=C#RGQ1P=@G*b-EiC+9#^EG>xaS5xtv*Ji zv(|jam?RyId?SntYR!L365qWm5w0!~IN!5U^eOW~T#u0=TKo3&oYcfBs6nH5fKwi;w0MoIE!KZa2Juu%F`vc;PDmKa{z+HFw z(O}SK`88iEhpcsvM`mI&b{2@F?wO~J{Sv&S#5K^Pe1s2m=?q~{S|w(Ekib{KXBHdt z12Vb#XU;9=2u8TFn=6YK1#j^Ncle9Gfa*X3$h_V$6Wj6HZ-`&tbuoe z2`IPtEIop8oH{cDvtLV7O%@aghLHizdRfR{S#XEW&90Rs-M3Rrb4z<)P|=f5EDU7&)t<2 zJJ**|?e?#uE>KVj-aVa&re!Tf#%XYp?ImuO#&s2r`z4f32Bdz?Z_j9?(mAEy`H1t= zm;#AXc3@}S(oCA&!qVc6NI1_xY{K6%u*lkt2*FU1~i8MUh6NWfnf+gKBj$;z4h;+i)Bk*x; z*?p7`uA#7)Ecn!|XobAq*tLbUzN-9F-ZlfWD=SU$z;x9#K$0q^0KN!vGOV0$W~K(KYkP`}S^B_S79s2MAc>lzU@G9`oYVac12`@XD|$ zmufz{Sov$u?)wvO?~f}+`o=b^&N>-0gfi>-@KfBF<{_fJp}S2?W!^;$?{z-!BwaBX z6{b{Gn`Ny%IPfKYPxF%_h?4QXhL~YwI+nqqzrL)KP>|n*$F&f?% z5pl&}!Ql7-e<$5I^9;zEBFFT;v;egnG!w#{;W%sdw)|bbWiEc=`7*o={=> zE%LV4wwX++-<-E|(KVgaJMOsVr>j%$!%S7UV#m8!G#M=`z)2>F0Z2~A7z9MF3tQroGS}$q5dd_&(r-o`&Y>bLUP*lEaBzjhc-W z@Ndzq_bNDrFlzibmM^VZU-;{zC@9tmko5?H?G;H0pS`pvJG zrA@+qIjayf?i)k9Z$Ega@yy>=X6zY9yY#h>5q3G?h0!0Q2|CvIgm@69E0rd(;?cc` zZtV6?TJ@ct!AQ`_2;KHjS-N>abX+SI9oTTRRfUIZ#-k&inL;<2DUGY%b(FqK9xK(! z&r2wQGvv1x+RYK4>aDM0Z;U%e_U3wsXdRGXnQj`zu)Je`;B|3In%wWKh6}H(K zOn{3k%Q+TdsCB4rL@&xw+B+4sV?ql5@2wjt9ZYN+A$?fjp)GW++{Awut*GTjmY9{&Ha*?$5i5ZF^1YwI)#u&=**{Rt}aM|-0 zkL}*d-gtF4U?R(tCHz&u@0={1HTN9;XCC2h6W2JVS zIcuT1CgE-HMbuQnvoJ?Sl44(P>6Dijcq58N?tX=m)}??ZQtB-t7pu}^4IZ=PyHi zl|$WNN6Xz*A6(S9kfEImzbTfABd!6QY%R2s99hw7$DoX6>MVF%a;-cZ6x$4sHz6A+~rmS0xkn;Yryv z(#HP#E{Xoc44%Eb_h^_F@Sb;6maxmu7>b}uzrgZS z%^!LWu3W~Pu?;{Ai!h&9rswmbOW4xielS6WbA|{sJLlf+FUCt?(OoBl(a0>6AeGnXfDG5+BnNZYCI6nJg z2Ctje5Y_|sZAxfn=-^RBhk^=A7mcSi|APO73a*1Z=%EGl0|NIoWqI!nteht;-Iw}z zAP$1Q@H;fgy?SYYZBKw{L?jw)NdV6&=$t)*|TSpSD{>Q(oi-#mAx>`gdFLfQohXT3w7{*?Q=~uQe{5S*$1LbSnzPo z=5+g#@(a42=XN9W{nQhGm)zeu7&H-EP1d{+Pb_t4fH>$tFcB}-t2cy zt9>)IVqA~TxcT%t!@aJF_$(UM4X}yeg;aRm;5GjJCS&f%{qKM0%7vn<5JldTRGByC z+lMaA^c*ioX$Xr?9Fof(uw=I|@5J0z^|CLJIsi*-_bHg1e+z6+Wd~fwJynGk>OI5M z3vSMW%REjk*2%@5v5Y)_mYqGl*ct5SVHxRV)CVF{r@L1#h>HIZVrgtjb(DRcC^PCA7mkwsl3`!j5gxK&rIXS$YCY%Zp7tj^s%Xy7q7*!`x)14nL!?!nOYlF7%jVy`4nUR)_m>;y>-_i%?`Zrvs!wo1MH!al_GUmNKW>sY`DU0a8;zd5F&*$L@ar)!u}rXgh;*s)*b8CJgI@@U zeNylDNzV>;Zs~^h_W}ylp;#AL_AyM-Pbo2V;zYZ!|47g zeEQOY|Jk>_?)RTbrq!S{S0wTwP#L{PyR-W$P3E&V2Wh@H$_qsFtixVVq42O${I-oq zn5yW!70_R|ElLA~I+9tjk#qi$;Pe2@On?UKsM4!D?^S88RI#|gjm^b63|i2`#@t}p zueO%PYr(*XkFG86fNWm8{6mQqyDpu(Dw({Cmj+@IrE2G%9JzhCdMkaypkTBIYrf9* z^;4dIIWsJQ>TV^#^-c+B0mnXwurF?G^}Gtg-}Fv7&@X>1!Y0N|wc6m#>kyvfj@hX- z`YH6bx$nc_`4I>SWJ?q{ORZGLGHmAz{BD)39CM9+9NNqh2lv*-7nVLDcfh1kWVG3M zA=%;K2m9<r3e~5FSrx$C<-q=PDRhY!5#qKqe zM<=(uJ#iufi1x^G`NUe_^yt+qPA?9bYkuy>BN6aY9{JJ$+LkZJge7b#kyPPOoBBB2 z7*y?;Nef6rx;#_A&*^Sl-wF^uwbJXB#_xWur)2C>z4T{H?hX?gE-*ymscN!f$G6YRc8@rqk+HHbw7N*K zF=^&d{SPztE3+PcT4UN5=F$wxza?WSXGGhIvCZ;fjcXWoLkT% zFr8zEL~KHKyX?uzdA+~njoq|96o9wVkE=-Dgk za^4=$wP|(hWf{NOd)#oFpVr~%p1?e4rzTczO{Sgf^Ws=0asNSO*H;ht{7`b_3DPCJ zPEGccxtn);NMegKC(G#fGzmWYz8al82nd~UC>@6O39CViG6R*-_|1U$bUyOQ&#p{N zu*wCsgTx*8;5YvXRFucB+!C8+Ak%Q>rMcu#c=?RKKU^o_V;_>etVSHvIbUlXGipH< z0zs(+BjZiE%Y1$;D3}o`c)c?X{qY;fW7>LwiBHPyg~8jHJ4GqkS}e2sboKjFX{hX> zAesFZ8BD2@bgf~!LNXH~ksr(@QjgD>#-$8X87q}N-+Rj|E>m^T(~Q~+5{-9Q@&{%lD}A;*EUjov>jIdtDVZ!H4>3*R z_yp)hsphZu`s>{Crx72OxP+P#!`hG$Y(+L=Zn?-p4B!{Fxk<8MNN1gFZQ>sPBB|-} zx|%>U&i4jm3#e(0)dNb|J<^y#rdqCn8;+_y(``1EbE1&fZ-)20q=wHmOEL3Kztz*o zzPRoy*>rdYR1#WgrH^}#hf!gLw5!WLzw|7$8D*t^fADE=U0OJYm)Vp^eNIo)_lagH zS}_jM<7KoED^K6Cco*OWYUduEx`c~&*pa-gLbUwAK4iFT-TPQA`o)~>DX5>eDI$}< zFnIr)tF}~wuwivfb)Mlm*jMk+;r>;}fm2MzK&AYQ`es>L0siiKsuEt8&Mj9%w>mBV znn}M0E-FR6Hk<(oDnvxCD3|^3ob%zqiW=&`a_p7it1G5{c$#nGTrK91DRsEK>PlmX zQ<|*q8Wm29?Exl|^=qIKtqtzHFCKe8L2}HMV!Ye^6L{S|SVdne-)!}bi@YI5|L|Ly zT;9lvR%k3J9fWaxJsd}sDB^^@`;eJ-DhsF(EQWrWp7`)^igxXRrCPYRGQQgE(G&#u ziAT8IK9t}`xz^d1)LLaqblZCNnpGA`TIz1ae1s8cW@takRhIW4~_iE zacD8oKcjc>O@U!*`Ra9y7CoPF zD*&H#F0;HCGbU+u-}PSXXij~h_jkAaU{$A`@4aGaj#1Jjh21`Q|C#r~hN~kh-O84k z_Al`VzhBS1(7R&Y!UBe_E}V^GZ!<|7-+V*54nEx@zbRp}--g1c)$F)bj3V3pe2ix5 ziHlA0l&Vm%q0m0I8jK484>M4^xe9*vA9AEsj{ZJIoUUWei+v*OGqs53nfbE-<1X}h zrX0ES%Sd&d;)AEW;J1Ypyg)zF{flV%R)R#C_w@5SQn*c^p#<=CtN^wt zKMxV#tcuG`F2Ynpgg{wC(1v+(cYW^kIJfL0s5`MsR}S$q4=<}Ux07|q^SV>X)v;6Z6^XR$m^@{6@Ma~*obdPV-+J{tPDdnbw z?9qZ^Zw=%RNH);r!SzK{j|~{@gl+yM&Z22QekR<`YSx=h0^0%7|84tCwsF*Q9C9)4 z#qwAll>gK7^W0k`=5#9sB49xXA$;r;7x*K38`L0f53=x9Yq?_Hg=qqkM_6!`mDfp@ z*PW$`0}oa2eh>b)$7zwIuzr60sq-=);EO*$E*Rtb1ob-g@gjlinkZkjO=$nTH)Gmu zLA!Q=9QbmJRL8F1s6=Dz;lvUTG(~R@1eBnNoIJOZSVFzHSJfOC#`Ow3V_O-q{RFC( zmHZTL6GK8ykt3)2pqv-bo_!L#I=-oJ7#l!rA4O#AUemsVoL-CcQ!asqSQ?GZbrW|L z>o~EyE53YK9^Qluo#jyHf;rE%#!Hw4IrJOc21(tinTz|62n6eL#ncQ=0+m0;z}Duu zDGfK7iBI`QR+h`0IU;D^V=PkNN9jFd38mnm+=xQykPJ`ZQLa{w#1*w*n&!?Z&~e0a z0CPr@-24zNPvOcXj3=9edDsmzPGG+0`|*4nO(A8eWGPvCFR7|M+X0_ztublO#33i< zKqbEYb4UUh4g}WI>UVdUgMTztenmX#1G-zP>1?WBPGC(7Jie&zd*~ZEQo`Ki10pWe zWfVv%y01$*#k@HK3IcZXRn;8slHUPN&V=h_F9`;eJtvwhjvqIk<$^{vg6+$jGmCtS zNY)E3*>du!GW4+E&f$RsgSCo>s_N+Er5YxJ8L7zfOvG;c!emuQwJ6(gB0;huWi*Qi zk~i5v!v|(sKh^ShMEa70$Yd0KzCn->}noo5m3KX{);FFQE;AA6Me|Xd$0|VOde?b$YczXAFSPU zLa8HQH%c~{SPWsTIpQ9+Fu8+Z<@6prUp1Rd6jZiHQQrnLGOT28B~F#c_uMbTTry>0 zG||#dfgA$edhXa1EpWnVLrGD02;ofmg(RK+84&pS#H^&*cSl|ZT6Yf(00UmO3TKg< z=5x)^Cg?g!0Pu6OY({>lmrYPI6FCUmhu)ctVvO-A=4E5hG4)(96`+Ir9TgcQV3xLx z0Um8c9!K$jvNV@KIXz@Jrf&LeW--!>sv}b~PqHmv_Gt>3Y^Vs9HmIk!v0Nwr!nZEb zwo{LNN?y8&>X&; zyk~NY*SPNzK0Yiy>=(H~0bd%XNiR?qED9nKui17}+|q}N*S>3S7z$-M8p*8xCaoZL z_|Yjx?_H|8tE`0z$Z@-d2wW}ywPB{RS~A_$-Pg@IK>xI;e^@dr;FVR zU4sUQMQqvb24Y;JSMvw(XQz&K?g`X|OjPNdM^l7BB*iYZ#eY(qWz}eJnOSI9*@pzV zKEIs+esYcQIep(us>$j}?NSO%t^!|het&{8@n@QZ3&)+4EgD7<75ignow!=^a9m)& zyZzyV$|~eHDjXihQT7$%PNtM0&L8#6?%5|}(vORu0FF>0HMbRkcrwk8cKc}*< zf-z)xG$7-Xd~e?Z5Rx`(<-c2(`ZtI2492ukXL;46XJA(f#Q`@kVq;}bBlNUnD9n{i`@q?<`0ym&3LH_4y)_XVm6& zRrX;ie~paFK8v|P4}ky6r8pn{x*P4t3H`TN=$SM3gO2P;u{;_Q<+VcmC zrM%`~%n4Lk{MuV~U_5e08CkzP@+IK)Z+CUN74{MkV1@VBR2@6zgb(PhAr%^-5S6>X ze9$0&>ypb`yK_!9zTsshyk4{s1iDFi^3H~mx)Tu%2G$Lny{Z6(I=Nbmu&G|HXXJvc z$o%OYP(3M{Y>Q=Wj3Klk4^THB#|%J_`| z7eIm`vS*y+*=b*@FYj0gb=VO^Y$^JDu3nI02a1raG#J4WL8{)Z$_ldy8R`&%(x zYjiWff$L9+#Vt;ExHH!^9qZE8diUra&A@3+e%1~mD${odqTc5O9DwMm0|03X5B7YJq7gghi*^U~(PP|#cIT89%9ZfK=*ZqGDN>0{_#*FHnF1s)@ zu-wp#zEC#TsV!Zr-}mFK+iU6n=|wO{`dJ0P@XD~me2>h==7swdsm2u1naa;>si4&C z!BTx7Bk04?)R~>RF&y?8QP;8%FiECDiJDozWEKphEk&Tx;{E<3-_C@i-cs)AFs@zh zbZP1qG~{fII@_3*BxBVW$>?qc)9@S#al7$j4cq4Vy4M(5^TT<>L4Z{dJ{~_<{86dFF})=Z0Mw&C^J}oj@{a-L@km%1kD4)h|+ti%q=6Ex^-4t1SyYJcjV&Tu%X`54 zBUOS%4Q&_)yh^`CroQ%Z*|zrqbqRqz!fiN97mn3V+AiwV(P>oG+k@-LD9J?n8k%Z% z#seB6j`vC`ijL3NwPXY?!mLd%!Ew!7s=)sKuhPp5*A_hpM&UIn08AyO|NPP*ccF8+ zXji59=gy&pA6f9owl^jmO;ajOQrW8Ujt{u1vjBYA-Yj}4BLTNG5!MqU^?JWP z0vNbBSO+y7aY3R3`;P(lc@hPj=8OZPqAEQuU&^ zRR&%TumlZz>GQUCH2&S&zFtEgskV>!FbMz_b7ft!9KJZ%}NY9r(p z0+d?yEO`}s_&Tw`g&u$j(@()pZq|V0$*T;#9cBwXK>`=Vw#7S66-&O21Az}2XKiV0 zaT4Fi5{oU3muYSnJkR2F&bc%Pw8&ex{yc;hl&7b@owUxpEVGQZ0;3@%*PAuh^1HcO zV+MoDoH8)yfu>KY;+lF#f`W;EiIxC8=$-N_2x^r~c{FA21x=E@HP#3v9{Sv}@}8C> z&fUf!9cm_(Vju=og6H2mwLo^59NlSU zHf}|_ulO!04E=*Sg3!SBB3wi04SQ8o7h}DE(Bm&A02PQLa6=I>B`( zJy$ekO7+u3PM!rt+XIME!*4Ot!bRztWA>y~t-OrH2EWf$9TscXb8!lJUKERU-HCWq zx#0{s8WSf$^Aw%Wmgz0?3*{m7P>!mth5(2xHV~_Ki^A?&caHvZc{UHKw4r0uH?tkB z#lhVWq_eagfyxbjqxgcDvh1+#iW)}CJ z*~O``iI{Sge>>{-T>O_SGfr{B*KwbAxfZ}lv{;fTV2JVtWH9IB99){_&z#G2^$b=Y zJmk`|A$)Mkxk6gHw$A(RoARo0P!*`Vb=8R}q#7)Kk>6_`YsL8J4JfXI`E4enij9d` zUNJtZ*LI2a8jmZ2&?9tL$4_Mp5IJb>h5KWQcBp$_*aS*4bqn+`MC1{9GwP@KH;xX$ zGak<@*Z`w@Nak?;f|mOf>LVkUhJuivO*!46gJ}fg&1h`f0=s+dKdLe4RC^u*-@oLx z+tnI3*c)(Ie6mb>;}R$SP(5X!R#Kr*T6~1*YV4Xhf)D#UsqiFyb24hdB+_E8MbF8% zIya!F(X*(j<;ZUNmIF?%yXn2ho5@?*?U0;+o)qST0Hb(%krA=mQBCM|q~1&25=$qB zRCMhipA(Hs&)XV6%}0B`9CG7D6bvopoW){1GJY}9R%>rZefOUKA`MgC2}<}Yepl?J z9YBhX#7kYSB$fwT;3<&+oYhH<3R8iZ=euA4o$U7gjU2|6_6&zOnby!@7Cb$?KE+ zXL)+FSUcV7VwF;!2TbMBW(3dY;6A9Em!15kWLK*-^HhPS>$a9djg@dOO001vR_+Tl z=(XxWB8ls}s_J5XhOQ=@!YZa1`LA=CADxw%^N{jt5e;REuEj>bZsQoN18{eVLh99Z zfTd;uQ`yByItB_VDz*dPf0|J>o^(ot(6xScUAdTaxf;q7_ND|$P%o=v*$1aE(Z^sP zq2}k5hki6D={{rxxy1kkHze$e1Z56Vv8};G}QJ=|SWaLiC6)x-RVO|6tltyxa`@hp;}& zHb^XDN}4FFt&kk5XaF}KqP8j;2n-K0Z|!&avD^p1V`F;m;S;GFsph@~*ZJjMGr$$v@udB)0s<#2$-ar>LiDfN z?9Qpv{=H+F=Cg5OIbwbe`rTqyQcZz{dp||KlVMhVzu+%A;|jikgWTFD{6N>`=Lg?( zzDX|BRpW2C=o%2-+^JH zT<47~cth&|jJ)D8K>0`oD9YNx^XfE_O=}QB!B~3<5CT;vK(dg*x*P4}lH7QR-MDf$1H_76Lcb}A8uQXms z&tKCetzJZnNtmiwoARMI=?hf2RFi8Bq^gBtB!IKo-9H)m2%8AlV4e1ZtLODH#YFtt z!TeU4=`8iS8JhlE=}o`4SIP%;M1ISb>a=QQ`Zy?rERpNx3o{G~N=1$m9iB>q)I}O( zDg^0KKu^4Pb`M4JknSDJ^XddE5U4*XH!fZ4=fO#-obk=6lLMSCCShs|f2+k=BuKsg z2w9pE@FPGs<_*p-+6n@+9_5voL^RZY(z)ZuL<)iiaC6Xp#rz{7X5GlxzM#m`d1MJ8xqr!gGkKiG} zGd=$Ilfb5~g)g8S(u>nSQ{#9zznHV#Q}LKo0XRQUObzPkbib1*T0icJkTs%hXpiG6 zEWhTLocP~o^|fN>1}&nih0RW5Pll$1rAT?>nH%ZLgZYl90ul~fD=eD#FY4nyH$&}+ zxxO;I7euaXVPIKej0>-w$d$l z9*1gG4OKlQ`0Va%SaRDoymim&-Cxjv{jgblxEAA`Uc;?t7m+jaL^ zlxGV5%e0%aCm8T8LCffOZ`A$?4y>9aJHz16%_T`a^4nL=^jHHwHvZ*PxI^uD^^W67 zt{{&Ft}GPdji8TJT=3A;P$pa*{Mc%AXY=Y%NTA%|Wqqg3EkAz+D~1g`mN{h}s{~6qwwdWvn)VKq|I zXxP0=DGf4cP{!;lkF;p0WZ0Q$z^=MYXg~N(UJaU3$X0A`|UuX3NElu zUjx~kWelwA|0Jo)7}S;dtf4uRguz+D7Qe(1X=(YjrQ3fCelB3=o0>l(876$}C6BZj zanjF7gvXBFbetKm!{yu)MD{JU<(iowVAtP-{PEoZwTU&REPV}evKI2@(8wMVU`JY! z&jiE=EFCS~Tz}%fHQI3<2GiWKj8ZCj{))!=0zC4R!p1%c`xBPw3@{im5-gJ!vR<)j;;d zHHflrXO!o*&x>&EjhwpA=#SB?sI2N2T#coEUg!evu&UT(O`gPx3}Mbl$!XDRmPUO9 zZkiga#w}wg?N56YCgx*(Zuz=V_@{`LaZ&Pta&N5-x4i%RB)5NCa$GRQYDL32y9}5R z-g@~r;R|u9{d1ieYQG8ofywcxj9_(!f9Qjr2(Q&y3dIPj`Ug2jqs;!ZJC|@z?SWno z{MKtqyYb7Xh7SLdU?GGG_S=$bSCB_tp=buCrgOql+gf&?4){_trjkD{jLz1_ZT@X1 zUv)FL{Gle~;_c5-{6jZwDmJV{j7;V{rTTmAgpYtmV!GZ8`*W{qu*;cN{vX<;#K>bS zbuEmnnf81?ssGfk$s~+>vxM=V_*9jdg6hfB8{fmdK7kG;=@wimG7GDn_ zkM4i-K*BGPRo?cfWNb+S2sLQVxrkVd>g?1Qatj^Sp%Tz55SMdr4|4 z)0SWcAuX&OMcVpL!{0QjR9RK?fq+k)!Ml5+iH>XcHY|lJg#6{ zV!_^jourF=dpZv(=gmu=4;H<0r zlX(6=69Z2_ba2xy#&4%rJl@|0Twr>#xQWxnEedJ42&dVCE_d( zZIf!G)1UaTQyS;~2!R}8g-B)L_UFvQm%r|~G4yIw`U$+}>qzilzun7UAd>DwTv>@% zc(i!76({|=gd1jJ&&G7Y2^GdeJKumgUO^f09=Y7)ihru~1*(41|3aZ6(;yfaFvOgH zQd`zE;Gh1>9Yj(qL0hhDa-8>n*Ogi<_o(aJjz==AE|AsX`jUKtw%_FRI%+zU1S&Y=c&Kx+@E40 zo(szO|L@1hFel3LmiBn&PoWEU*RtDG_>wm)@o}~ThG`=-`C~f;%c`WgY zxQ~xLC%}Nyo4QU2m#)7c^%aDX)OG*=MhZewZDC=OAGkiu!aFy>GCfb!yz@>S!JfG= z{Nc|S(Ojk(Dl~TaBA9)+YhRuBDQr*K{Pg;Hb+v_`RsG&amX!z@ZZ*6fCKBfUF?EiO z6pSo6~WC^uQ^F?C0 zQ%r;np3Svzx1gf=uPG7UJFnE0u@9k~i)^(@-`{%g8=SDbAOPrCBEflik~v8BjTAry zK5(Bfua$`2A){Z?)N*&l84hEo=inO+J^I3SVFU7*s=P5EVD?DaSu;?tP$uagdq$JNI zE|obW{AT^H6hjIw;Ihn0Twn9^dL6^k(PCry!3flo1KOVO1sKwubbgPs?ebC7j`9t6 z=?G%weX5i?{hk!Lt>7);;uYjPv4%YrhvedU^U8t(JaSg}LsUEg4!Q;{A}5Wv0W>C( zf0AObRFO}HR97}i>nmCSNM%R5u07j2K_s&F)^%glasf zBQ_&z?JGI1nm@1}bygQ!5Yyw?@mOXYB+RheHlSQs+NNLpKc6ymZ=EcSZuzbr*;Mf< zCAGJFX~DFbC_$5bzSepm*ATIEc`{U^?Y^=;AGDLh;!|O*-~r0(FJhH#{@6YpV#3jq ztVeBkKryCf(g{_rr(z1Z^mO|RjD##1XD9yRSz9MSp(>(#F2;mxd}WQ_J=RpAx3s88 zN<<}OF7(^eYMIz0J|L4Gw3B&N*Eli->rB9_Zp##(E^A}S!kJI}cq>9DNmf0rjC^4> zExWuMpx$k4?V5ZkWwI$};8}FjOmCGc=rob%C{RK(wB=}vr{&p;VL09F;49I+MM;$) zV}Qokof~v?zHfFWjo$={iY8{dKrT!e_sV2MYHBdGTvCHqHs-t-zv)i!j^ZhTv5N^rOBI45Ld4NDT8H4>CRMY zep1IJ9C%2Hh>*6Z4aAy>_SJpT1m*Mhu7Noo4Ew1N-{Kv~FpRPS<_;*sE>_(Kon4f2 zNgbY)BGG+V12HAxkq(-*j%#QH(?;3R;$XXCNe*UQE;1ntPN~5ahqfJ6r0h&pOo3Ws z5OZv4eSMR#D)ib*o+@D`RxX;~X*J~;kI_ou32wVWID`!ZQ2nm*>#bvR3m#P)qIMAJ z2eNjzc_={_1*xjA+f^9~HOEZ5O3>Jo@$nLk%LlaNzPfHNw}R^j*m~^B6!aeH@Q(E`H0r>)eJEe6-$dw5(Of)1Fem zgcdCd^7}s{HKvX8j=}ZNg{4~PJZ7hIi`qNo_-ua-!ak^0Bnjnx@o-~SnbRy$`Fhe{ zhQ?zmV`@$j7u_e$Z01)tup8mCNHp5D?bFheuy-{n%fYiS#c`*B?Vy&I6?>4sq`>Y$ z3A(FEjd%x&%(|b_A&Qqz-WjrGKyDRz`7YhjE;Z7>-CM^3FDAbPKhK~c#+`@RTt5VF z0V0@#se%63bGwN0qT`F$P#lHB&wF7yGLx2nR6+@g`pIAN#lfK8(gr(AkcL{B2FnL> zp9zYPxY9l0JRJIRb#598C%$Csf_*J4i)nJ*kq|ooW)$G32uAPbdT(hgxphG;cvIbJ zUnp}ON%Y6GA1G$G?Px7Gy=|H!v(8MEn+AB77GDR{JSOXWV$8ZeSC%h6oGVZ5zz-@g zD<=c+psDzl9pCVhV=A?je>CY;m5fYH+j7wwv*H}MV1cMNTRnr_&rI!IOr7j&PXpXk zV6+Wd0T3(uT$c$1WCBq%VHVtGT{8fU`tR0Nfa;I$vN-!vYOwO=q!NHyE-fO+;}`MA zLC&(}YOBi6%yZ^CjDKQ>5xKNko|`EijG}d$^mUd`z)GI%o$Ry(NVC?CSLLYz7>!!xj3f`u?{+qc1MhBw(EiI(=te$h>&(rx~<2ntq z59`_nqO;(d8kF_N2rPAjW%3~>Gns?*p)l_ow9I6k_f^+J8}W|6VW;Li9o zf&!{E%!`lEJ1`?~Ev`IK6iBA|)HNI~E!w0>EnP6%aauXtdDMT0{a_d(b1CwkW6U!G zzF%fzjfHTK?NFUX!Z>Md|2y7(xAf7IheSkLR^r~oEO@`oF@ zmCc0F`0XB|yxyRIAp?}7w|-%8P6+J2_1rLv)Sw)u=+RQ-pcVN4SbOt8sMjxke3WcS zi&V0;Ckmyq&m`>&l|mRQNhRwTyCKBAH(O;Z%WzdlNU{zFl_`^@^PKZKuLDZ9brn9I@!=jknokij9gNA{T=C}B z%H{W6ewzv2*M+$qz_Yi5gPga`)qF+=n7mvSTDzH8KOpb z32^fQDr)SSxAqRll@!7|_x7un2oW;j%L7n>fr4ZZxn}Jz^X7dMb{~th(r~s}OOf|Q^*n+iILFau-g~BALI-=6e@GlQUsIo8m z8zct(p5vq3TC$=SWmu#1mT`&8I*_E+`DrW1wA7{4bsC+s;X{Ha8!6Yk8dR_EaM&vC zPokJEr4=Dc35h-r%i_u&(sao2-yf{0$qN>Q*Tju-jm|ka5Lhde_A3Zc9|dDNf*KQH zGTJb9%H^30K?S-IXsT{a*BMk>liW`E>Od?EQTnMvFw54CeGN*ls+OMUgUroaY3Ydz zT;PzdiN4cAFvUi{sj;7EJRx@uG}UR>juBka8hU(@350pNFR!7nI4TGOs&KT{jb-n8 z%6xg$gvI^wb=g!B%erSoAZkk_QhpLziZwD8GSaaP`gkZI>-hV?@vCXl!y7*CD{-d( zS>BI02${vw4Mst>^(5g+o=oR!WiXNZmYT$1g+RuRs(kb@tM#QlcBIajBOR$|v`KBn( zErw1p?Wj`?oY2|o+_KLS^F64xtFfpYJV&*L9^T-c=YQYaT>vn%+etMmA1$NZc*tPs zNY%b}iE#mot>+$n*&*CE<}QDD?g9W=WuL6B*s!4#45v#vERAuXlo2m@X^eRR+N0b- z)ztQ_)O0e7TO8r3KqN$kdrQ}*V4Hh)r90~ActRGP}w#-crbEyvS_KwA> z=e|{os&k(8six{KO>;eq`{(R&_g7l8f(_Thz1q%V8bWFJO%E5NYeGv*IYO@ z*B$uLuC?k|mBxMVn}ePeCoOIU$2bgBhKF23?$uHSDyu#g6dv2wC-1R zh$T{;R!#xH7=T-3lXGV`CJ#ADTC%hJx8=OLzRG*bqw_n3%0@cm{aAK4FAM`>8u`|= zktv(>YIzKIgr1utu)9#F^qq8lKo!6*@h%Ox6V-;X^Z9;lOsx^-fw&>Ln5tT|mvXW$ zVF$RpXpBC>gvHj0vi)W^ks2#)R~I9K>S94WmSY{R z5L%VWztCtJJL|BOXTPbz(W*7*ckWb2@H*noyTJ20GI(ADo!82ntIDQ-U((+b9Y@8p0ap#R6 z5%&gfru%vrTd-?-jP2lr3kG8Q0=%OJyLuHeVw)zuexRVmC~!FQH3bPD`e+|j!Zy7H ziZu;C7m+L9rQYv9@Wpg}GILjSzk4oAHB|@eqWLq^qRG{}DzvRwXRx9})KMq@pu?SA z4#qUc<29zvfN8w=n_0xpXWvpVcT$LN(Yqt}(0_K3CW6#Hm!d7X0)Px$F0?i-O3V6A zoRn{)+0%#ZnQ5xY8vWT2gMH(Qhn@Lk2#^H z@MT|S9lxHg56ym?C$|cX2K~*TJ>o8MGWJ#IdFrMw ztIBk*#yV^j?&;Jvp;}F2birytsC7?Q1g-3m1nKiL3f0BlX9ShqGSFz&udWX&d02~& zHD}R%r|aeGDnVzTK0mX=UqPJOzJG&1V{;SbK%ChIIrS-TC;!jbb(T!2=6cE>YE*~B zherY=i$)K9+%%Wfl-ElBrnV_Lk*daCk8E;LPH9)X5>pY|JPI16X_6Iriw}>p4(uU> z>>Y*!ygO)r&hY+vGR|EPQqW3u^sm;Xm7_QQ4j@a#NuuP*iWZLoe*Kc--JclSk_4N% zMX&}->YW}+&Mt(?rTc)wCAcO4uddY8FEgH+`@{3bMwa8+KXoiVA-C_S1G=zbqMfa0 zb2f@KJ78_XA-_y9dA|BV2hGOugPvO)asNZSM?AOdj{FKLK?i=jSfggSI8zIYf||$A zyD(7a>J6{8aU0mXI>SJ6G7EPQ7q!)!)H9{y+%NiA6L6i!Ho9f#4?N;>+65nt!1yF@ z&OHUj)j>tCA zG1SE7|J>`8VPh2kV$`h4LNK5(M|lcBe+nxoa^}1m%HV3y)5(X16h3}vD@k3QpLf9f zbARcfS|XWIu$I;rKNF;?uL=fnOFbl(w0%vf3lELj2h-4UsrTmT7HQ%K+-a zOOOLQ>=KXy($@e~w#K{z1Tp3pv-lD2e&ILZ=s3i?0WB5SwI7(?xUkntQ1XjWL6)G$ zyJ`Bx)*ofC1#pvlYaOiMm?j|%6s7z7d1%nj;DH1UF*15pM;R6)v!+=x-nzS zeX#V-qLS5>skuJ|3jhrzs>F5V2rh7^YSdbB@qhpzbc}2*H9YFWW5BF+Z>6uB(hv)J9va%5Rt5 zSmlJOYbZp1Wc#&xPS1)hIzOAUKo5k`fV%n2z?FK3HBL+bQXsmf(p)AO%PQ_r8S|*x zRvOt+CyjMTzRktM*jjfC@?V*0)tWbcZcE-n|JKp_VGek6mmAqc8XZ7lUBaFkBW7!V zX;{&Qth+sOpHJ>q1)T?P7?^ZYy|bikIxsZJB0NOPH&)&~z6Mt$;L4iW2i` zaQc7&1h0ZC*wVufJQwGMIZyQiQ^lFrI^%U>*aSO% z%OpsCU5Hped`a~ivY|38F1#O}%}5`)4D{}ey%f^@{s4%nGLu*r3NXM~b-{uZssfhZ zpHv-hJG9{y^A>>dTK)jkIh0QK`)mQfXEb#$KKG%D{bCt#$kc+%tLV2pB;sxwjlRg;+nF?fuaM6B+MEjRvxpcLqIM3&<9I zc1pkDdW}`EAe7&>k5eq@-qwW{w&WO$r^}< zv`@JL8yFl^tZrMRtMTB_aZ*O7|r~G!=YgY>I(7u)>880&?+*ea-jbb5a|Dk_Mpuk?Ev5D+VC0)XWug9CT0Guo z|JTn6%9cvWiH-LLPkODO#DWmwNuYLzgVb~P>)#=*jMG3;QF?~WbFSXi?tWO^{LU#Y ztO=gHj7|YuVbO`BMy$|P@yB`T_PwSrl{<-@ab(2$FYTq*e{!jB_hE~=zamO?g~4gG zBGu6WA-*ypg>L0hLr%4IQU@+9NRTE)Wp7IE{JqSh7~#55cbGv9DFx`c7J3B9<<37CI`J72)*Twyb!oWFX3z2Yq zrUnFQy&uI~c8~Nceq)%C6v)TK9*EOO)OEHNgq!Y)ed&^EUs|u4_x*qe+NK>z9ToG* z>Pn?X;)V_53{pW}s6tLa>CuVVYCFvAGuGQX$Ss%PC85q^mO>=8WI3;7+5X~m=389o z-cH|8K4)(WyytVE8kOn3F-4|T!};Llz|n!Pp!x>Rq0RfIb)0dXcB;EKe;w7(2Q@lp z`Z6c_2D1&x?Z{f3?e*{Az;hB*$$$4(j3hy6flUU>>elW^xHI4WHQ)e<^YAtF(7Kog zmwwwkSA-7xW8?5 z2tWRP#j>E952HW4+?j2Gj@c)sg=$xH9)9P)7?2v{o}EruPOXlQU%FGny!XG#a=&Gh z`P1Mr7yx<9?#6A6BBhtGmSW{(ChkgATG4t(JY54jUT0qehyrTB%=sbAxaGOTJN0I#Kid1;_vPo4`sRl7`$P3t+1-L zr3xT(;m*+=!5&nke2i8zuj1i)Not~YZ3~Uj$~Y!7(7JzcfWfL*S>Wucuc6X5s;-|2 z=!#+-wr?@nx!#n@t6ed|tcYY)ya~0Hb+D2>&PDbSoX8yllG!b=l!NZ~X}L2fgEd;E zXdhi#jZ&`{I;C_U8G~`Bn|p{xXN^qFF0WmZ4_y^ufBY2ga08_NJl$2TRSbciYvvDh zRV~X(EyIQkD8P==E7TS-&}=!t!rlYnMZp%MO;*6{UMl_Ry>*-Ft!dD>QyCu-xbIDW zz}dtha_uLw*0Syj5+%%*k?ZZNakL#d1`;^OHW`oCD-Em5-AiL{5NotQeLp$N47@jV zyzZyepjEznXPO$7@1u!}a)U0IlD| zJx&g$I;mlp1iOY=t_2h4JJ8bNd;#-7n4!8@*N@lJYAm+`_8J0_4f8AHp!Ts-T zBsyeHbF;9q%igVRo;GQ$QdeyH@s=1^wC#dQ)+^#LTGDNld1Le1-fWg?amk45si1q9 zABVVEulcmZEnXT)I`4;O({qmmBfOAPuhdE_&})x+I5{YFCh~s26Te~%^_y{kQsU&t zcdP?UTpL%7M%>TH?~KjlyQZdjDEBgOj|qODy+F*w?#3X{S%v8IM2==QYVQ^-;kmPu z>+O)%cQ?MYx*nJ_wV=E61~bZaR@Ym(81EPJ{)}lri8?}x4j)Fo;PX3b$%*HAbrs|d zi7LVPxcZjD(WQoQK<@N?`VH+D5KnU%?29d3Jy0VjkcY}!PV`U$_+)TiT-N}#8Jh4} zbw=(Z3D34n25VOAbMetMZ8AYQVXf2^FPmt8obA>@nN$_U7oMe1Ar1yRxwh&9e21on zbL8gZun68Z(~JH1S7>EP0H1;VMdoaZ4a}(E{sZI7wMT(A))V zl|45baKgLaMe2GtnjU}=PX=rSjc}l1{)Nj)6EEUQg>ZE*=m)r;0i!4EUP*WNt%4sP z>K}oJ>(ol6@@)YibXn}LPGSu*1dG&-g`dFe22ke=_YSl!sDe>e|7kTo35SulH0x;C$zwx zfK5-2+Xe6pFyU3<*T*VN01%2&QZB#>QOwKBS+ne+#o3|7d8oduZv!?8s)E2^gh#li z2`O7P!WxYiS8f=^pIEN&tMvvU!o;zU!$77eEu6<55{Vdn8Yzf}mh2NA8IY_V+RZcK z31`uK00%0q8R$S@QApT>f9kh^Ob(Zs9S+)eVG3mDHxr&_Z@&3}0~PGHNb|WZA~VJG?@>rU=f^ZnYBH?s09!Si2?2S z;Q9;!yEIfz;)_8S(uw_S`kyERd%yXI!xK<*e*flWWF6yg=K!|dX?dej`eF@0)=mYY zEwQV9p{n9Z8GyJNkc$&){7!1Zi0y4vwM2YnaafJ;kUjk$n3;jshGnJO6};X5)NZ@I zKn2tmrzE@;BrieeSIk7Yd6F#v@8RPU1NfzABPqKfuoYr|?oaII7wCV8mZ@15r7)0W zfAWo^l|P?wLM{R|;0%m$r>5=_qsEZUKoF-Q34#rntX}q-)_?>5t**ceg*Z1vB?b9U z$j!^+8+pFc!moK02T-XLc?{8!O}@u%Rq`Yojngy%Cp zAYnHYW}Sf(KD=uuiftE9EVYj%DO?EF*3Vk&{%TL2^LSiY`0 z%JTU^k&*D$fx4lVR<4P45vZeANW&=?<%rRRVSLD<#;QYAJ^_i;e_&t?rt>Ha4~dJz z8_Ynz`((5Uj0V{NK+(p+RrYECSg|1Z$3QiT3Zs>)Vah+ZiRe%SN;uU983MEzKs&wP zj(usESz#D6A!B3G$2bNaQdg?;2B<_N3%(KhOD{q{N!+EMa0+IL+gH*JX|X8HABS-L z@KWu+P_Kcyf!rowGsO#l;w1jz1DK)PFYqQ}uJ0O869`3~g>YbPThE=`4kM&7CHbJ9 zj4z0x0%o2Y{T?4${;3jv-={TFO(re(N*KHohB$hPf9$BcC^4J~QCEQ1>Trvq<^dPu zJgIt7?p8~dj9NTQ)p@%4#e#Zy&)oQxBvH}CF|V=j|6D9x%SC$X@_|1?waLG-8vp;t z4c)@?Av}BQS3sW9-gojygE*+cV&46{w?9AjM5(A|*IyXy;pf)IlXFo}rfAu~AHCz8 zemuk{1PCeJI7ZMo53p{_v8z%OA@laGd2~*4zuNDxOo9=g&z=W7kk+2%{S+$KIV}+l zKmBXZ05J{o!{_ZLQ;qwkyr|y;UP?ruAW9kVB#XbF1Q`g!P{!M)pBihiQST!X!KySF zGUE3z<^S_pFunmk=?C!6-B}XD&pyhsWj+)9J8r|gJ^x3$N>NS)p~XJZHYc~i@)ZLx zZT#V{r$zAufH<{ez{N;A5w+6n*_a`A$1fqkb*z3MYpclPa)iTLwOf8Gmk+B#{T2V% z4ul}HgN}U2bvBWcFvL%zB!?W3KSoX3Lcdz~#fvuRMnH^DOIo+3hFN<-ii)-wZ?+Nu zXx4t>hE0Y&qIk%lWLx=;7r@_AKg+srWxZK85+sZ11P$obiwP$;!H>=sv#Y9lww(KW z-CAXLXQ8TJDuyqCg&jx#=Q-P@wMK_PlM#u)$C`!&Fjkyj0%rOL@PyG&y9kCtBJZRrb@iaxM!%Dx8pAaJj{s) z_4x37BixNhO{+jp%M6kZ}LRN}RpHx%*0C+ExO~zkDlz{5@}L(WLU`{d;^*cm zM6fr;*E=yqB0-TwzA|NWG_vq6wBX}OdLpBLxVQ`1>%U!$UAgPjOP~hQKZig+&o71M zjUPU^V%unF_0qG{2HBV(NeVgL-|?VQ(}Mjq_%a>@+O8dApZ>e;Vh0q^rI}2pVM5G3 zW4Ap0-$i%#{dC~yqXa`RuVl&L2LBc&=Gb>8=!HiA0Noa6>y|GKf>}~qeanL%r{%$m z`v0F+Y^j#Pa5v`x;JwKYShecfTkTmxWQ8|`fImU)m|sesx)J^b#4lHV!JT1`1rM4) zoKSXg(2O#f*uZV=g5J&Oa8ikN(dRXeV| zfEf2TYo6g5b+Qqr(*PrCt#oMxGH?%ygdMuc!zYxkwlVuezQYv1HX=*dg}xo(A=%;`dD zyHz-El(etLF2 z&FMX&U3S3YYj325tHOi`K2ObrhcWQ495_?$wB>@v@eqzD#TZQ=bht9a#sN+kTKC`` zSSCI+;3}h^O&QF<$qV?5 zUq?b{tvEH2+SE2*7_b>e)4f0ED;9}%Ck`2n3-d!ljr$DVvofZ2Q5=SKVEfW#^b$Nl;nFhiP5B4bPJQ$lr$~-Efc-se)lmKL#Ld4OMZP3m z3l(p*gpYl_m~FlLm#Jp{!&GhOGpwmya%svAT|(=W)oJ(+I2{IX@)5*G0NIX#BgdJ( zVZP$`6lzsFB5-bEA5%H`QY#Vm9IhC*)0EL&mjG zZNsPRa%%@?@b0kTV~~19zktyU*xL7hFkE|0#Mz3E#id|S1&fa7@$vKX?$>XE9QP|# zLSZo9OUUUjnI|xYu{pslBfZ=V{<$p}Qty~Q=cWRBKwH-&S$9mIyK*RFS7m#nKDo(ALZZ^;?LDD1y@jdU3IeEgi`MY=7%y*m1|K1pa8 zpkW$3mztzUp6DM660~VW2M!g$9J9G_;m>K?d9Fc!etuoak|v=B$CaU-hdqNFaPy@I zb_OSkCHr9arlFz3BV?C=wfIV{C{{S96oGO5X0Fj;wUOt?0hP)bXg|We*+3vdMx6uzROT%QAdwColns{$FdE@S}>RqyDQu)ki z#dx_!bSTDmispviD0mC63u5V2&AtXq_AprKQb+e$36fXOs@B~Rjxl8q#u7D>v2x10 zlLn^ViJ0Q)c}+?)fX9|<40#F1ORb#Bw)EdI+eMhFEETm)-;B4spl*4zc@{r)NlUXG zU1%~dr?$+IVANidSw7^%p)ILrxQ{$B<)Udh^oxz;|1Llz3l}}c<~NK6som5Gw-uXa zH7Q~&)8<|i2acdT(~bm+UW)k*#D&KsGB0CqUTJhFp=WDiarj;rm$$^XApPyoG$!Ib zlY2t*dTTk^n9cu52%muS3CKa)%14D|nuM+G)>>jx`^%PX#C;mJJw9n~{IwEMjLPKx zd6zO`WfImc=<1MYHUYiZ9yUkZ!_*a%hFrs1(a+cmd^FLJ=EdjSu7}g=EEhRvsbv+Z zU-2YVIMU}zbdO7sxMWD3?@@;S7-2NpW2OFdGYTkSlr-=?)Jkp5_CvgDnn7)Tr1GHE{L4!926i6^+OO}fSlNAt_UdcS;WQ&1! zudVMsUhmY>YfRg3xo>nD&mm{fFUC|ksCAeBlnf)JX3YY4FisMz!;EBntf#@jdFtpZ zLa9P=mFT5B?6XrOGO$B20{vG^cv!4n*a7WA5m)0e{WnN`P*T zlP7~zhC1HbL(By#Mm478gzk`cMo7_omNpvjl|UVbK+}a$EL_YOpv_V2*k@5^aevSz z#}JVrH=25Ay|*XDEj7bC)ZpvWFfC1d`&+JTD>*!apCQ-DrGJ&PpfQOYht0`eW>z)W zVAa}3MVrr2Zecge-KV(~W2=n6k^7ITEqZ#aET$%x5~7FcozlV@L}1Xld?Ua7pev?vlRL8+A!w4u%j&SSmfH;>Gut?ZM(KO#zZ zYj^)VVy1xny0hM~JvbwM>Kgv0gX7}n38~0l^h`Yl-iqE!*cVPt^q`mZ^xp(QvhQNT zv~tzup3+c}z2erXgsC92v{5IoaLSS42F5Ap!Ck~pwSL>5Y#P7!p3}W-s=uPPEiv8! zaaf4iNpx_}@L>>Yl&4MrlRZ>u5%cuOjmxkSnt+z6aW>&<&AQeL0ld{-*`|JuP%14# z`|rc#=f-5+_r+%|SReske8E>OyV#;vmeg~7EZ1_xbyKy%Y$|vEJ9X;w`=GoskpO5( zb#SXWJ;Xr+O)TxTicA_AuVU|CvTwcn4JN#r)1H)5bqVQ$|;;mXRCryQM!$d5g$73LDB z-CJkdZ$2yYQK>x>g+i2^Zl1BoeFXqOf`qj{Hiy3HtK;339t5Y9QUuXPnyS0?R5oK) zW9T0C&{7P87iJNL<{+|gFj4PUQWFG1{;ONbSs4ylBrt%cU3SL*ieasQVz*GMu%2AC zkL;>>4`JhCJ{tz@D7<@B^oXJ*Nu1vjIf71`LjsM1)BMBSn7RBjGc_vd9Tx&tI-~fy zCd?}J(cZDB+|!EI?LkI+uKg_Y-##*De9bArL#+k-Y~sMXOy7GgEoSt`G<@mKes5kN zJxTbOc6UlnktD1>Ty_1`{iIMWyulgAsNm+`iuS-9n;OKo_;Z)I!l1pJluNCjuW=b3 zgKw4&n?|Kyj2+O+tOS+xJe=9le4vUJ0CcArbUQ0QQh>f)EpL!E@)M~}v%3&F}pQ!-ab<6 zuxS6TkT~S*^{EQSg!5*eI8P>!S90>1@%kJ{ zZ+N;M@T|bRCDXhokLvs)35L9(!-Z;XV-vDDGndrEm!uMEpC&$dM}HHh{@yM{5!T8n ze7`^;lKO&v${Hs=NY0P^QS8bcHFoO*u&dbGKWMb8*f{H>L|3<&TE4geu-TV~GKw0o z2j-Bv?w)RJlPZPPSM zk>S`C=}f+lXG^4YbFvE_OFBzc2=H=bV0Z{AvedqheU%au0qwjWisoghLu%X zjsC`T@)xMf%~-DZ$h|rb-k>Z&5Ej4)KJel4^ycLD+8uLdT+Ec~zUia}_oNU2C8-lW zKD`|5tTkQ#olfy*T@<|+WJyo&wYcJWk~&TNEaC2qZmiLu?NB?Gf0R||x9yO9CJ%54 z43@BYe!Kpd`pAE0e+VVC?fZug6owXUI3(%o0Tk^kkV8|(=&R7@}lRga^z7n*#|op zzUu0W+r%`?OVQz~{)$$#8vRGmCk5=;mfaHX!1O7?mQTgto=YaNC1B#)CuRC+(m3$+urj19vOb~byazc5A zsEk~~{tfD-Y2|d(pmDek;SPi#O2l3QXqL=3R<}+2&j$=Au+CP^dTy};Lys!X=66Kr z0K&4D9vIM=^%cEP^L2%x2dXru{N3Auvzsz*JoHWT!n*YIeo~Wd>h84h(jJ0@8v`e|qFPpwu z3?MBf9E<)Z;0!Nmn99pBt>jatph(2!ri5jG?yj>hwD~*ZteR(B2%pZTuL17|^S(T} z%^B19gB6w;iIZuyTJhcC&<4G;1-JLY2Co2;x{iy0rx8xn#f=UDCnry6D>glOcKMaN zS+YAQYDf<-Uk*4WCqQV`puTce-B{DP1Vt)T2&>3p)xB2D3wvMVLL5A#>5Q=A1XDu` z1j1gW#3|GO=n*Xy%%>{J*zYgL zC8@aIW#v%2Hu~u=<3628;(T6A;~G?~N=q~#ra#l{Z%eNOQDt}7;i`82x=e@09mdWk0Mnn)VfG60crcWdd{>ad4-mGd2w!zeQlz8*(qsniF(6D7={c)wh51O8c}b`iV#qXL>xE z>Of1RrgtTx_Fi_fJX)a9zGns1n#%v(sFW_!I>$jd>bTv$bll7+&+$Y}LNRwEr#e2X zD2<7mDqEXOv)?)V?x4(gxrxp#{?NTa$3r`U*kidDS5%W+jy#J*{xEc1)*IC}FX}C8 z-?1MoPJ}EE_JVOLiMXh&%=lIo#X@#d!nA%v#vH#S`#f&N`IzRj=p%R~$H{~>*AM_+ ztBxzwnJvE7^_Y=&mqI&tL4EjOyH_fRzzL%^3mj+#-eE_>=#B;OjheV&GEI+T)R)=6 z>H&(V{VL9k1nS}XHoT0?jXjF{ta(9nuh!4e#oEi0Z4I=%97>(*MI7TEyqDQY;k@mj zmB=ig9f|<+E36aGx}Tf!C`4nfI?omkGOYG#I^**IgqgI|)5H?6 z_|JAb>kjr*pk7B!3?y)`V$=$r)wT;7@N!2)CIM=M>>|-DX;RAMaF5V<6?!gHpW$x$ zvgS&H@bPS`9fw{^zXDq%gnlAu?_J^8%xeSlgD%H1fMcY)go?5nQKYrL@}G^aC=Yv} zO}&sW@)13^lsQUw-`26J4{ji-Irxhlba6^+bO>79EHSgl31fSHP}M}Yc(>!NtIAcc zZ7C9U3^^%~YKMor&ImM%}RQrk?wg*iMYAiF&6-x=K{7ea;SSJuRZ z(7kAJA@yyQJ=?`t4=>GwhL?+J)d~*Xhc+w}nUHt)+LX|9OLlDeYac?3aQmX7i#DRblbVdx!0%L)lK4yOTH6v*#KDwPg%C$2L7;>$u<+c|J+PWmmYF7ppEqdg3 zf2<#{jOp1C9`@H#7<>p=V zC65O${U=-D8>Fx*aZ}_nC3sQP8SAD>$@}*7t}BYVFPB+ZnCI+lD#-P5UgxNlRtCBw z1CplwW%9d65gR5HQkdb#U!%5_Grti#pujstH+#RPoITYA~%R}DDsx1u*=l{}p9m$!9qT|#-AT0duU z{2y~(r&%Xj8;$6IxWdhd6LcjUA#}lhm!k6-uSnFhm6MorQweidZSV~)v?r)4av9l? zhbue%?5xvS;bA*0bDlCna}093ckgXnA!>SYN~V|qJj$lp#Inx{D5r-YCSMOFt$N_s z^+tBC@e5p5TffKutI2 z)fR;;Q?sUMkzYld*KRX?dv;?F=s6~qqzh}uy%t|t%*m3vh??(REfUdKttgW)gD>!V z$>N^Gt>#%}uF3t!JTnMSU3*5J7Z$vyv028K@6xu=U0C=@oqRvCv9oQS-!t)!vtEP?tbBcXNT;rz=#yPqWjpbL6 zr)V`8Pr3q&C+=Z8OLaMn%mq8NPCK~R#LlkX&*&q)YSp6uS|3^l|9BAlAnQr1gWynu9J9_|H1cAfLVhCP8wbd~&~Iq);bF;NIfj zo3P_eGfhP=_pJgl=Y-UUB(Aos)(%*^!yR;YZIjPh!-OM?+1#)cBEuxU>S#y)8qm%8 zhns3lPdOvPDK?XsFk0;SgMYcN7^4-9@b6!&Q4TJld$?+8BHroLh)ZOs8aD=rgp5W} zW`clP{RLpI+owCC6OF2h31T~2|23)0ba+tWJQ;m* zN=n0DD7dO17aA%rV_JXv!W3`(LagXq?-x6joj~T{S1-0-5$665yxqW+q`gf8#;;8- z%PQ(l9v-qkarBFe#KabU^!K+a;@)EqZ1OSJ_cZdL zDiuhK7A>Ahh-A%Z1BvN$^WoLQ2TZH`;1r+cF5C&`6{Eum$d8}WArE3(%y|*5t5Ur( zKyAm_UW2XU(ZTtV>>5gmIgjDG#gLfZc@eZqHBBI+Sk%$>z_Ch~L+>(oA_-|WRXKD_ zO#*roF7!2!YY|R}WVs}AI!58U#~RvGWc+IqYZR4U?vnn824up$RHlA{AqURNE9X{8 z2|*SrUbWG+7kw$0{pa2K++ zP3~{`HifIzFwqoq!>6Z@6N|i@6U5(b)xd@?Ra<-+1mu|ucgU87&+NTdeWTbIt{bl1 zSyw-1n6#$)qDq=<1}{6nTOMSJzIem|MroM$6?pHx!8voqfa$KNo3gX1=Hs>c^N*7P zl#h9#zcOBRa2{(#adAMBl*mQd>;Qcn$S{t1p1(Xn+M?!}9%0P{u%*?8P|2D)ka#xj+KjClyh;p4w)-I?gHg`X@KUL2)2A0E?7Z!2B8eTF z!ht0}(B8PDKzWSsNvluB$r1b%%-?l49zS&COowqJDc{1 z-@X{PqBAYdLai488shCoot$_%nbmJ>E@1BR)iO3+3djKnzrXgPqGp;=m}fe$I5YnF zy;13XNx{082XDaAcAGP5XvdCHaG$HGAwx};Lf!1;c@S*RvCf7Z<5zRwfGV*p@}?|L zG*7_KqPYyCrul|-bBxXRCyebfj|c~-JGw&en)BkU#B2@Bd?MFAc4)E9d8no+wC_&$+lfAtVtp`;!yh0MHsSSU7E#=sMx^ zS0NF2x$lT@{%!Wc5MKOwR9w>VeU!}kK|7If_KJa*vg{1Sz#f*iXDh>q5Se(mpLW6> z!3tCvw=;wB7LMytlm@)iBY@oID79hx#ETQ>PNFJ@06r|D?o$%~NcJEt}sI=Z} z(*XPdxZ#Xxs9*&;*Xg-QD6>J?k#=RKWEF9?1e<^03pTu6jk#C}G%(FCLt`^{v1Z71 zz(LWB#Wr%WhR^$EWGN==iVrKUZD}da?7HB5W$ESz>X)rG8sz7Uv8Oqg`U(xwcPrK)7cuD$t1o1FUZu_qD#oHAP^i!j3eS&5ktN#F${9w{6%Qf@tcVF8C^bR$FOK^Cf+@| zfDRLNu%?P&U6gh!&4Q)+l=`Z>U%4w#g=sy)_bS+3hdo7-+WXs?7R@s8b4RQMa{sBO zLr47!MSl~&cVRj|S2&-J9d5ezy#1v_NsIfCmCb#R?S6(9wur^WDIHKeIspjBVbz;> zk4qS;Iqs_o-nQm%qx)lzHX*zFOw5@^5&|3ULdnFR@QO^vF(71g0in4g=(UBSbWAFF z>YH=o#bqoDwJC=tR@p2GWjbF)Q84EPQ$Rv@HS*%MHXkc6E~o7CftJYm;?pu57vi+9 zhDE!Z`upBM(9ru=WOC`n6Ty*=iH0Q8IIIEg`<_O0>&>I))3_&}-Ipvg%o*?+!yMxr z7yDl17*$%!Jl+|Y`Ns3=*`veO#0(Vyyt{Su^h9Vep;I^CalFA+Rx>SNPgC}xmN0VJ z=_Cw8n940{r=NhbM!#%}rh_!1Cq43BEANgjYjgkf3Feq4`NLuBG?ZbA>Y z;rtC^K+CQ$sB-D1>pb4A5jy5}BF19f4(l_r{|&1Kc7D1NV9_RCxxVR5kJ2W;)l$`P zBj*D?Q^!bmG|YnOqaZ|~EcFjQyb>@w`%yA>2@Le#GtbEVL6{dVM{ZqH!8yIJ)hi+j zN`r;NxAvLR|pWFcf6#nt`Gu+PTYN=(%xbbVwXBWlU>>%6xj z_S^NriAGab>Fc)FVbz|lnhr92zp;VJZX_HL0%dy+1L|d1W8Y~#QOar)&&qZ{Yq;&- zBGq7G{8c7u`{4Xn<>T`~wYi&>dq>J|W6hU4)<>7{QCfLE_MRppyeTbgpI$h7?0U(& zo5|pz5#^6o>ratjwLEXl{}B!6_kI1ENb0x!y;sglOzhktzE)oEwZqbigk}$bF_E%g zGxPuu0hXDv7?t3+Tu)0i62eFQcgGt!ttrZJ%V3W_$4>4AU$o7}){fXa2Z`dKgq`<> zMN4?tMwrGKpa1z`y)esMfOJ04>I1+`KOchgAwn^UNWsiZe04Yx`9Hc4HHne~*tO=T z3D#%!xGpJeqw?B+smuSPkIbU}zy~e@WY7!JyOOP%%pjXHFEFeSgpl=Qh}Y!KuNUma zm+M0Jso^OI{j>ys0H4m}H!T=F7zUUu*;W%v@ajnJ=ep#8OaNuJ7LJS}?!-{T`B>o$5i1;~&3^>ng*i$? zfGG&{@GF#OoH@#ZN?>7u?6HwOSZf)FCDh?4SlxlCe7qPR&il+1`fxu#Kh$!o-4gZ@=uhF<0_RXsnzN{hE3OW*6vZZrKQwFt zixNTnA8X59O8WpVI>dKV-jAr0iG#+_4gq?>Ups`n^dd^I8_$j+&p(p=y8E2hFU35* z=3k2G;vtLQEs0P~8iFIbBSPt!N7I+!dusW~m(pwy{HFGd>pW%{_heBlwf-a&?hRZ1 zTeHei3nK)v;7Ykv-ZZU3$(jFGh8sd2E!;;Xu9TaC7q(*pRi6J^R?B7xoe;MS_dn7& zqn5G2ngrl0`a(xl+xJ8lK-khjV1?+y6vk}-kT8!tpEh||3^d1WX_Ff%L6CPMy?Dhw z{`~8;jsITjpIU7y5ENzR@iJC1N>1Xxk_8KYt+4pX5Wg@CO453VF4gB(uP0I8c4uz* zx9X%mllwFjGv+K1WtK*2MgXxQS_otxzgk`oVgk-JSVUGAlB$?#6*2fpFYVt3Wic-KAFXK{ zZ#xUmUjv}j0BpD<)0JhjvMpr(SIG&*|Lj%5kG!_|AYRkaX7-5(HIko!rvtzu2(3#J z7PKjRLV!Hap-gKTpf>=k1{t>fdlojFNoac`fL?G3cnXW->E{==Pt+^1V zdorK>s6DyFZ0V&Q|1HTOvPP&lF@w1Za&SsZ9Y_jHaH^KuHnTt-VzbgM50hU|iQ(q< zUvNTw=P+usyZomr(w=B=;{Qi+rc*A2(!-7(o%_`&Y)(Dfy0|>Xaa;c9sD+%n0*#YFZ~6ws;6i4xU1Ffy?cCEhqxy;x$Nn<) zLYo&Mp^>v3_+g1fz`Mv@LKtYRdtRU$?{Rt4uf883P^a_$pO7a#v3!_;&|e$#w;aJ{ zd&Sa2;!*N^c_HMtAfhqQG_>>B#qf@t<(oj}4T9LfylZKbK(i0R7$SW|^Cd_#bQBc& z3j_Jb_4kT_uR6BWk7hRniXRkPT&ovWMJeXj3{i>UcMw3LPeT_ma4(!!+%~RqVnsSJ zMN@(O|50`2flUAZe^N<>KB*izI#Co6IVb6MmMb(Pp>i#Z977kKRF1i)0|}d>+&1MC zwaJuY!!l;JIexFVKHuN>^L_tQpHF+gUa$A_e!X6g=ku9i?}) z4J1QV*-Q-RL<)lytL%JmXCGN=`-<&8to`=?haFkWGDHwt$#>7zBymb6m|N>;A`7I>3gD$g$>w9&IwF^)Jn*t-FQKt zDA@Rm_|=7{BmG?U{}~qk|8C3kPu^GQ%4c<+d$jM5OvCgigJ%h~WK;f!ParPdx{7&c zm{MdrGIugyV6m(*=|7&lvs}>B5(8?cLvrZ2xc3ce4Z(l+f5wnY?>C;^8)SH9`_5`f zKp>G1w=*X@41UZ@f&1rZ$Rmp-Dm8e`LOjU*@0c9HXZGS-%+_KVHkw0hsgneQyK9y_ zxr@J){2y5`ba7fMG+Y-6?!ErHMS16|zqA5t`lSeDW9NTAbO3US;X=$8un#H)`2(=h zDga(v6g;$7Hf;Y7(Z+}|MU+T`YK;%mOt6yJ|E?rL_!EhczzB^W)9Z!cZCGfkCSqpi zZR{&Vl5)xklLUS_s=KYZML6h3L9xQA?Xcs6IO4M5;lU^%FP!r2ru90sk-U zE~aX9n|0e#q4Tb}XPR~_DMdJhz}Y?j`eHk2#~g`{_joEYJh zOcAk?vWQG2QU4|syVelq*)fwb^8ocLFcn1L4u#3p_;=^I5%OBbEbx!)NRN|SmfuxM zp)tarD<~%QF)02kstpUAIl1_;SZe$Ke}t^y+59^Y+UvNWYN69Npv!}}?FlxXqCe(4 zmRm2=Y{Pllo$!Lq6Ba`go#1+3#b;pDI1ULSG8^}RJpWyu>CE|zW}Uv6V|Y8yq&~|A zM-ZproW$?=#OQEqh1-|E(5oe=Z0&TPHq64i<)4HBCEwAwZxyY@9i0(AHNDsbtMYUJ znDp{mP7)GR&lrVg$&m`n27QFKvFWmQ6S_)3e%J{pNqMdl0vafQ8451FuCX(u>d~JYS^Wjk9 zfbZ198TcFB&-!%JbW3Q*JFy4Z+^u|AcB{J#0a5t#@vKf_?CBu%&=eH%B(ueJ9qoo?AHV5H((->G) zaiP}ibGJT5ii4ofOOdkZkUGG&VJf&%KyleFBB9uKD)`EIv()fM7)BW~jH}P{w8M#R zL>$;L92gVc-?lvPRJEq0LDTvIk)QtQA&yUk&K)jjWQ(z+%8QyTs?DfvioJl$_{`N1iv-#B|7aEMNw>H?Gvjjs^=kDYiuy9Bk?X76usTG9KhC>9lDarQ~kmi$1?&hYun2iBRB8DMfg(e_9rCdOMlKbLt<)bEHf3Ntkgg?AyJ=F*V|Bo=7G zIt&4yBvg*jSiJtGUy0($>#4=|ed=GlX0XIHu!}M74?||L->}%FBuNSgKU2fh023Ph zs+14=Rg$)p!@8eL-Q&XZoNM>3f*qDX?y`zmAa|7I>oieeTWj@--QUT z!~Up+yCVi|6tBIx9bnN^Cw{JLXPrQg!J)}E8IKxTx;d`uHUB|~u#{fd-=fKLO`v5Z zOnDs{ET+Nxj;kAIFoQH+>+5Z+p?3$nz?&VrVv^8rZcm}TisOz&poh>`3l^du0Yvmt z{D%QMmj1Q}B}i3g=JV_1VC^lJi>r9Wkpo{N+bRub_+PonRHehOWpf#}-u)i~!l z=1aTYpdpTIDD`-t(;+&44y5#Coe3xbSnGPOc7DP@$z=F=*42p{jtMa_S+86CN-)>e zHE`s=FSE}B%E$I{JdQ1j+z#+*#HYs9LFd2%TWQ#kx;b){P0Gx^*d_x|C(Gkyz>B&$ zdt0KEY=T38&bDODn5sRkCfi0znqmgdn(+XWsA= zW<*-s;afnj&u78;#yQ=?2U<)i*R1+of|30Kr9S2kNHQ-e`{zV`4cql6D%{m5FHFwM zww2x(+-nIN%3zR<0J2RvNPy*|Jb2nOxcZJUR>U)!&42#iZMDU3t-mc*NJP8YH%MccA)eH!*{m zWyXx1m~DJz87FixMWtM+ckOrr8G|}Pp)Mv5gPxLPZb?K_@rF}_i$vefQ>4_H7?9%v7 z=abP`Mz=6splh!zn(B7W`*ZlY8Y%S{B%3*oR>MAR>G)nyspPv@pWL#meoOB9eS?FJqvq>*B zO!muYE7z#-;7ihr!adrt`W0G_&6M)LFmM>hO`&c?Ll8}|#7o8wb0Zn;6n7ZnOG z#bFnd5LDcG6_X|o?||lDSU**mNFdzc{#<^%(Nj2T4rEI0bbe5xeibqOrMfEZ4U>o%aUy+gzX_r7vjyPpNS zRwE%{_4f7c{D2(b&&8Ginrj2;`4?toWG zGd;)eMfAt3$aH4UvRd5M22^CfRkBg^Mi}4CGd)Qa)EpSshfP?sTF4$g*By%y%(mJe}5G%hr+b ze^`T%OC=zsN~V#V+CDTg%BZGBFt~3yuv`Za9)?mKgwm~qs~d=Ta2yb_;m}htSarzQ zol0uB9e&(%i^s)5r+%W7Ty=!`t!*dof6p-EiyqvXZ5f|x01OYdZ`{$aGo^NF z^!t~MWo&8V2}e-$P<){>z*VzAk|J?;h}AgUzL+xXYAPiJmGy6hf(Yjbtt^4ZnECeI zx>nAN5As!R4VO&I*Wt=0(ewb0-E&&)2KEKpXtf(IVW*TMi_Mc{^H=d0p}OxTll#sn zRx51yT&WaE7m56=XRLR|n-dwXW8XD@o=Fl8%SF&OIn=5Ck08uXYU2)<}t z2y4u9@qF=IopaPxg$@VEh3pwa@~f@8Pmb9FnD<%PeeX-k8hM~pv1HKWHSOb5uf1~L zNGeZBKTtO(p`Vnv?gX_c;|mOT)1KFrmG)m~FWETo?$$kYOShn=AHdJM&^I2peI9tQ zlNR88BWhC1ziP)t6TSN{6pG4Xg`KOD0#2&V7c#@nJgJI$jAfZ@gCkgH<+vD2^s9r& z7EkGG<5QnOE|+4#=oCxvJ2Zz~_y#vTwa$zm?OLQ7jW)7A=BlG+s*A6i(BzqfwhmA` zoo-*4;mh~YtgXgnGuw~o{qXU-bq+J3%z6Mn{QmOTQot1>by~2-ylGf(iGM>>y=Tkw z!eDa50@gaD4@4Ne(2_Wdj04b?2zCma>o_prnbNrM&Bd&>>Mn-fA{2O z;s}pSQxzzJ2eIFE_8eed@a5Yw@f560>1LlNbb9AzDMwBgEyPd0p4cf2tW*{iW+f6I)eBQdfl(8Ti1XZ7TgMO#&!Sw@cDODH5_d8 z8Pk2zeH=e9P@wWwLtLp%u?kVi1Gnx0tbwB-&27^}r(XSA-`O4(!SxX8fEsB5UF8c> z7&y)Lvdd3?@F!lnC^B7>4ubmCRSB8GDNUx5Q4l?SqZX9%P-5U5FtJ|;Bp}u0ut5)c zo5`ZM?!12bfs!EY0f65P#z$y{2r*R8?(Z_B8+2YkL-FAqzo%BxVf0r{BlPWU;Ucmg zTCkyhMwEBS%*!ZezvDG3W2<6}%pkZ`x9^Cz@bJLsV})P{n;9BA`Cx~GqIQ;aav<7} zuG?8>qyNs;K;L-V3xo?ppdn7li!=P7lG|eyQZR7D-O>8h>)Qe1F_pK{xS+3g-^Au* z>ad1r;=m^d>E;|}kU~N2J7o^~$hO*g(APk@e;`blS@9^3P*4s^|1SJUv}5>&jq~Nk zquh|`FQNld*S?TK7~s2;+5t7NeJB%=y@ZDDk-R)1#egu-pO4=7$-pe!lyTv0YL|rN z{=TZzgz5IOIdFX)teO-eowOz4uVl@t#|FX<1L>_(+G@(_^Al$yA-!qLwm|J2kf%w0 z_DQcbT?NDLaQfsNyzBX2jBj2+pQ}hS^O@oe)P~It0fEmB)KxzOG0NjGjUM#91P(gw z8m-aMFf&XHl(Y6A75fAjRgn`&7PlK1m*cci>A4X{zMcrj#X6hZe=+5aTsp|o-Et>$ z{Rmq6#|YIZk9GP{GicvTYCTXM_|C%P#RD+`lR-uJ)TvRIk!Ob_#Q$^Uwi_m zAS`P?GR#~$Vori_aG*fxC0Q%dce%#Kwq|1pEwO4!AO4o4q3VJ*0>vzY;}u$5vuBvb zri`Gt)G`Z+HFRpt%cvR>rTRdxCN^mkxd@hZ=@kFDy3_#L7Vg3p*ImgDfF(>-HVu=_ z_^T11h%8Fn1-mgUXEnh>b>5NI`25W7aOG$B>rN(M`W--SKwy{1Pu*J_^p3#d?Whl- zUazCDM=uU|R7o}_93X8>?DDKWDb|fJ&-Qv!s!R&%2fj$Sx|Q7GDt3{z=~qwM6v%8* z_Q_!)L|N!7k=7^z*yT?rvkbQCBRE-EAerOcm&wSyc=HKbYQQk)#xa;MY2l)Lg;SW35{=bZ&al)Qc?@38`*l9=~)U1W=#)tS?=mCj?VPR61VEe#N7{UZ23c$A0_^&kqyAGf_6Z(SbqdROzIG3)kXyd% z9?N0exm6~lMpz>bxsTWETLz77TmyY9nwBkcs_^U2w_y%gJ{|yR02Fku?&+F6C>-5O z_>0i36}BCZT(*<;Z~l;(ox5O`V8=b4k5Dl`Fy_{9*wR&rNv5GHOZ+q!;a5J7WEB2h zItV3u)8#1~UT_l&0!9SI^krn}@v6T^NJt@edi|2MoAQjq*J?BO?_%^wOx~_t1Hco@ zL|~m15)0KECW46ix_ULs#gR*{0fBae&t+%MSj-Id z{J-%d#`w1TZw`gLq+&@csp2Rt?CjvqfgvVQ@0%AKvN7=}8L+4j2eC^nhUKY0s>2@0V7D2`El=6hF*f>vC=x7+hvqKD!}g$fQ#yNrX-4}z{LVWm z5|tvS$dkEnB3nM00FE7QbGUPGW>kLEVAWuf z4N{Fb;ip+bRkk$b>|=fTqd-I8>;uV%# zsHfrvL+?daDc!ajzcI9`S%qak=>02{apUotzbU_?6(!xkLHLo~jmA8pAUC+dBt+DM z5>zLfao9G@osPCFh{g%=xVnHnT8Ko?oT5VYU7a4if;sdK)Rk~X+S}$yV3E;jp;saa zqV?Q4H&Ugmr(JvsrLWWMOLui=(uKHYQq+`d-zCu3sdU)hkupTDcGRmUtwWx3c!Pc7kz13i@{hxHf{oqg90mdu&E z7<4H`^KtdPoh?(KZ~7^4l(0+#Ef2t>|7JBQRDuky*q43rVi_+%NdsiZMb$-LVr zu@GR+dBL?Oc!UPWb=6yY2*}{YV0(BKBO-Yi$1Fmd+Bs=%N=7d0cdd`&QNt_m2a!5! z2RcUzpVLAsA0v=`=zFyny}Nb$94i?bVOL1sTmr)TII=WNpDP>lo59|pSGfYjctBkQ z^SG8Ln8@V_b=7ZONgP3uO=sdx9^LTBXAF;J6e6j$WfreGJ`%bZG1pSW;rm;;8@>aL zb=L1+Oq(0VP0Me!tiGI>-EOgS7Md0wf7U|f_YmlT0Gj8 z%5m-5kl!14hnN2*ak2O1J6vExbLzyYx^(I0(STs{zW4jqGAc-!M%L!cu;8{I&CS)i z;RJ3^T*7n-&4QIR3Rdy-q(uokslEWZr`QIrb%~j%H&0=qUj?~A?1)~(xA<(Z3*!%c zeOWcGdE@bVnVJ%kK2}rpW9_po1w$7=l5t%CT_Q|llgZX(cGG!KgPHiypRW5>a?1K{ zozP1X{cBR$d@9)bX!QPxo-pzkHf4x*VTXE^MNjEgEtc|f`*6SNtixJq=E7u_|BH#r z{;(G0=F=rDJBHJl`B4j)*8~E*2fpor|e@zA6HN9kk%=xYIAcH*gB9#dQ8KR zhePYf2sbL&J&y)rd^aCsm2q31JQvCul8mvrQ6olvD^Ap1b!_$G*U`^1a9h}%L)b~h zF@@!qtB0iT;I?Pj&_2U2yzDauLE3omPWH!Ydlg1-kdVH0)xClX;vVP|7tmO*PQ_b1 ziLy~PcC}Q&G29|2^-gP8;a;C-aEs>zJdO}iKv+6-?Zhg z>IlZnGHuHoc_enJY&j|Bx6PWNvi~YNlgfT{7>1fUsN3JfA5$%vV{-U}b3f$g8kn2lZ|dr49{iZ;-$>%- zsWvumY!!~Q*cUUyt_J}JM_S(IN(?Q4Ti_z)fDrC>i0;?nq}R zD`=v%j19jHdotSNo{EG%ZTv!0w#>P5gdEapi2(?TF*#A#)G-X2fKiqCpkT0Ubt89a zV=ms(Uf1Q~i(rWzOTULrZ{!6>A^3=Zicw7Ta#dxO;)sw@FdQ-d>^2A85}lWf$_Fr= zv)NO`>aD3%9(<4sCWTpfl{(mp_BgHL)VQ25Y{mT)q`c2vFiN^zPNfiSdElg4_M6$@ zWv_(iq+@pYw!_;JmIVv*0F&aR4|p62JES;lH>ORe$s(WF&b8e_^Xmn<XSL6MgNvV5KWs?*{>n74cGaL~D8u(&p}@iRA0-ZjwX&lKIq^)c<3`Qq(N zW7^gYM`7mx)Kdq7?2Vu2eJlZ;U6i4FGq`vANsG*oOrVsE>*c#2YNV;G#NF_m- z1)XjH8jGXdcPp#R-%_j|UA$DAxMw7%R4D(0A2N5i9EKVQ0J->C%6oeJ?A5$7uUL5p zc7K_3{xCaOXP)_Q#!Wr!$Wo*G(AY^0ry#s)2FK$AYv2hk%HuNZ0fzDNRNz1yHgsJ( zGw{(4*u|=>QzI6ASu!K_AMR~R&rLtGg@Ac)it2hH(?`W1IPMw$8rqBfa32=l7n0fCY6p2!KIKlE`=LrmJ@#kg25@wHgB zD_5)`lJ<}>5H}WBX(K#}QO-xc6otcJ9YN{y@9;-gZ(-y9J&HFn%Qk9pF%;j*S{W^O zANFNriTU)Gr1+C-ho<`DdciPGQms_BbqtAZt@#NsKTCTBa`wkMv%bqz5$SDV+Xoe) zJ(HEVdAG7kJ;=TZd$h6f>S-qLPIm>V?Q(w9P-O8Y%oGhey3gAG?*aN`m~|5on2yz- zjGzmZ+%dfnXH41?3!0o?^;+668d~GDt;tR9Y8vf=cTo}L+5Qi+yb9}@SJ#vUyhkE1U)%;pe;(RF3Hi3^ z>gJ>N|KNp_YX>pcKXAB_H|T*7VDI*~LxiPGd)^=4E!v2026tbMprA_)ZgrnvY8I0= z*ae#F9D-`@d^%n)*zY^Iy(xUtzs8!vTrZNlILM5w`eRQFmsO4iUWWCYfWK;s|0Ygg z$_8)G3cM)EZBO}~IV=aW&3V~c!?7~#V1(kmMGMaS*MyLX`bmAja;3|%6{W$terJRX z`md3tub?#~VZIip54^9q(}N7XO~Pp{?lk;rlqv!e`^wkONYrC&u->uu7${(_-oTJr9UN&7|0<;PS>bT*IgVSSmmOV4o8}TEJR)Ihgf1 z!{*tWTzF%Sme9YY2kUg!dG~B8!+H%Tcwyr9!eZ^7tgyOhm28p5b5yAy1w6g4J0*B6 zGN&p0Upy+;ToTL(W!P*MbsVt|!okTV=N=934OiX+LrsoLediUzEth`)tr4GYfP`OA zN_x6Bc3{A5@8rj6bWl*=!@gfiQcSCVueTb@E=Sf~6FlTC4H`hobNibJO>7lh-Ip_ys)mJy!G!_l~Eo!!-e;UP;&oL z>xqX*VR2rBMn(r170|;{yqoL6UAKp5m;D zCyW*DuMCiy-uC%kAoJ~zD%y%a@FU{em@av35eh z@Bid^7*{J3d*L0xTtg4O(|!nrJA0vwZB36KlMbqv=w$VaqKa}#lf_S;WH@Q3?@0PP zA7B2A&J3+-ICq=1>wm`DgEOZks^gSzsIJ^0z?^ox=iWIiJQ_LBJ1^TsuhLYD^-XSL za-+{3nMGbjs5HRiR`CDZ_f@<7%V*7f^!(PC7x4RMh0MyTsxR?h?8v4|!zb$A7f`01 zp3hC=iFJ-sO`k+*znVaZg5-ag`?$P4h49KeiuGWx+-~GG(f|EQZtt-f-ZpDe;4&7IBGzmDH(SYRl#3Oa7&%%R~g}%cvD~8;V>`cyoKGw?#gk zFe@$2$qSuhH?LLuAMR)_PV8l@N3V?AIJnuKD)ZNgdy7j>*<-kBazI>Euih^5sLr#>4-0abV|E!x=y? z>SJ&cV2p`Z>|gu)MPLdR$Hv8rmaZ zk@UZ{&}|oDgq5%6dvOv8dDmuZ&bLKabA~UasNYnwvcZDSDu@K*qXN2EWxqG5$ueIE{;&1-5Ik3 zMYHF#?+ohvrR5%%SCOoHpC0V^{N!a zw~ufpa&gfIQEu(r?Ei}%E}QiwIxiZ5oAIp4b`Kd%6xH|tA7QUMZT?K4BkRP2oSru* zR4}uA%mv^G_xvL6Q8TR)XkiSUnDB49BzA6K=Z~o_C%k@3f_E<=OW;tw42a!9gsM8K?&wg47t6$zmtfGp3wXdgsaM439#B`bN|j2L5O>T=%WXY1|@)i4K3mGTicvx<`@C zgSp2TAl-(e6k;t5_55(m8R^|3tkxNc;pJkwYFJ^R}i zB0G*QCJ>@I8?TRzpHVNU9soTr(qT!tB%e*NZbQ}iak4Gke9aC4`+Jn$Dpuk4D19Cj zkcD@k$m%|79L>hl^lbD>EH(x<=W6di1KAZfw z;8g12ZsG?O@H#3FI%mLBp{yi2XxF^%FSjr`(TD#^SjK6Wp;)KSLYdXeKrk zhypwtSG^U6GG_Sf<3DP;*BG(+Q4Ck-<7DtlXe|?#Kx|^&=xdfPm_;B}g7NIksoJ8b zTA5MK@E}E`aeAGf$R>960d%u#Eem}TP1mg1ysbMW$MaKY z=zefaQhW98h%!LdMM#Ok;yu5q!osNzB;{b^SS|yNJ}Uv`fyZvx0Z7nD6zss%7X8@? z%$#tnnBlHB8_pu6B20pjHPrsF?jNBTknn1vqwv$D0}*w`HRfEuQ}-90)i5z|wtd~Y z*Klp-TX7VElWy+$vlFvhltt!swADcqbU{o*ofel(F%w{@I@r)sw$Ttzaa!n3nn8?K zQ}CSLldd-(b*^0O<|xASe}_*z^jvD^=Y17HX=oXi5(SaIRuGY59o9iZNxEd`&EaL3 z4Vt%SdD5?JUTFLa}8Hy=)1(--BvE5)j8B1 zL3@qN!u5ahorWL4i3DgD>3C>Ms?4TUup#@A{=+xtqIM3f|4aD=CvV>W3Ai#iY$gb4 zCQY6tL5Cv}c}i!?q>T-Iz;Z)p<^?$b z(*nWTn3IQKZf~a#4J1!|ZIA?cS?{^;*q*=kiyIb*G!CS#Cl>@ITzv&QNX;thk=v`E zCD5reKdb>33d5&CC-JnAmCQ*4hD8+?F~+8czQWC4iYl=vVMbRJ_lGSVK%pbPeQAhZ zqD}lBT>o=`C{}R*T-m2fC)3W!4sk9&Yfx`}Kk#PzHhm_JR#6JE zW`ZzjKdQluJjEuA^L)r58Y`rD6=Xl8mk7dLiZY#ZXx)1r>E51)HA=QDfqH0Jxvz>y zu!{+nuc%=jw>YROPWtIZmrhG)#TO7Ju1GLAtosP_*%YDkVIOn9t+*G%;zQKaz%7aM zOTRwO`#k$wemI9wzwN_~vZ$95urT)AdUG8rH7il%V%+ol6D1skoWeZ!0%E|YMBNBl za{V!ANJ-ywr=wqS?s^#FRHCwB4GAMG@(ujfTNGj8^WWJ#7Z^$&R99P51#3g%M=D25 zs;9b97x){SM~;IbhScIdaQHRAz{W1_Rmj$4bR1AOD@p`)Bz*5Mg9n$ik1hwa(J@bB zc+DSV(ns9yfG$oN;^cxT$MqDVwr&E0UkZ0>Kuf=tDbPJou}PAAxfk zkUn;US_9gPAq5{7QM8W98Te7$Oa!7ev49wIV>xP_GpzBdv*Gna%&YicqNqxC$CM@p zIccJx;07nWK}NP!f$?H*?`z^1?h%1C4t$ci#XklgXqoX^`W8@No&u}-YEo~gh6}ht z99a|FQPI&A6=JYP-zYq0(&TU!UJ$HFfXavt<#KL5m)ZgTGTRS1G*cw0;V>8M7@leJ z5r@X-SC5Dcg-jaqZR0&+b^7R7SDHq?&r_|-JD>TL z3>|K54mrIosrSPB4N19|pNH>$hwnW6)}FcM?x(7NwGXjJ8=GQxcFb9DRBwyqD3l{WFB|9KkZb5gTv@U1V+IP&GbiN1rizz(J*r zW1QQfp#2|VM3tsV@q*=9HTWYmEbGgi$fAuf1qStuRj^OT&sUs1jn_Z1i_X!r{SM?pUeLN4QtnmYI@GFw{m)Fvb4?=l<=rB9g9>vZODp zg)Ffw(I%o_26bH2b;1XdOz?z`PtQ1oo!YR5(v@jBNb#C6D{v5nNq$i{5x(YZY?Fgc z!JMH~yr#J<+@Po_i2MD`pVweUNvjrA7`IsWXRLx-)~|xayC=MedbTUSYqh=xBB^jI zPTs2h*941wki_v&g_UY@|Ndr`g!9ib z6B{ZuMz}Sk4eKsBS48;gdXVTOe#J`s3r3k!k#@Gzvk-|MD8%ej8ZoD~P>}-;+M=*f zFZ_%8RWgk~7NZSmtjJ3zz|6G^?`zQv3+Y&L)X9R7y|$CZXxFh=NS1a z%ThqOEGVE9$)xY{zN8LAx%w(M9zoe6qbtiaw+x}8^e+fK+zJ9Q&6~smnsTBYg(0|e z(SzmCMxxP;(UhC)M3#CmaTI)2obiqJsGKt%20h)EO=0d%di%-h&7um~9QVmE!tgTd z`-yO|YfWw=t#=r_OT_q+702O}KGmbk{b~&iiu;=_vYfh)zup3NJnmWr(|dR3+~6A# z1;Un57F`E-&}ljTg|R~ZMn^T0%uGe*3jOEy8_5atk9r?_wH3%t_fs8vgv};b{9wPn zT39&uQeRb!$=K4sWYyIIdcnn~%e-|U${06P(_B+r&#cNj?R6%$j16(kq6+V9y|eh5N6|<9VJFS# zucDU=go=v|(;l0_L>tGYF0<11OTh{bu(@2g_1!bE!_TO%#?MUi5ma*HkD~YruHq1+ zBBe4w6s(<`;YNFA0v&<6>0dTK!C%08K9hrG5x0~pXo#We4`y^V3Bn8r!zhLE3bPSV zE_-EKwe-%X`#6ZaXONh_I|jJ;Tz)?}^XNJl^W`9^@YNdd^JgXZJ5d+$rat(%GUq>5 z*#s!Sul-nk2apk7s_ALBdMiUD_Sb$F!-|cK9v1KQ@?1zQ@2?k0U1;T1u5d>3JniZO zLIFjS;MP+J1a9J3H1BlT0iSAeI{tj}FZEV+&@QSZV$*3dcJAsbg()_M%MRu*22QwN z9r_)!HW>OWV0+(rC2v2G1O!4V6&Q1WR>rzMrAAk*O56jE%$M=!U8`efp5mm(7c<&Y z#Zj@O8Qm}05zp#_4k!>#(cK1%q#WO_NU9mdnk+aRK6yBCArq36O1|(k^L?G-T|V(e z)Ira^B0~!y*@tu(2h>GTvYd>5t}sXE7W6&QpWXF}Aet+@#do*~P~e)Tr;ELtTUf`^ zabpC@P`X(7n5tVa6wy#DR{x%)I~yN=n5*z>;!!iIKWLk>7zw-!J5@-?3i?{C)ZVMf43W7a*3tYd@xRH9)0FoIxxBDQy`x}+ z*xOifPPP|Njd%Ni`D(1p)1sRoqPglPz}B>xDoN(R#P;5u7w8(0n9g$girnA2reToM zpUPu51eMk7_;MX5Dry5AhTe|M{Iq$% zVH24WEA{6S&RRdEp|4pMv;Y+kea?otff;ix$j%1p`DV&6A(ooP$;cA1Bv+s^#;H$( zpFCP&f5O>Nw>hhzR(f9zAlU>fY%_Qb7TdI(-L}8+>e)ksy;0CtB|QOOm6r_oD-O0l zje3Z~MZyOl(g!^$^Wf6CTcIyVa9lraaANfFXU>;#v4U^lOt_Nq{`TmMLVX`scu&d$ z`pw*KyAcZJu#FpW;lqkIIHg|jsW|M$+nOtmQo!}VN)KR24=>1ppT-UpQCFpziTzQB z2(EQvqBDN*^7ogJT#V+gWDKM9l}uyMyd<5cKzuK(XuzJ4&TyN|E-Cbxy++52frl(T z@MobytjIO4t?r7DWIbYKD^t3*+kv5C5$9Ht^Yu4UGrC-bSD~qsLV*zEYb|?qM0x81 zAK;469y}cX_Q2Fb;mo$PtU`e2ic}eBvL(EG7tQEA(iMUVqgBR{;SMi%{@^pG*mRXxGqat!OFS-cUI za6X{nI_zuw#luDMj_cAmGxm~|WsoL=XCmuG{k?lHjA6L({de7fz&NheS4GBz@f}oJ z!cs;QkFI{y5It`Ixfejn2D@aLzW^tO@_oK^@z956zgA&+V1{vRK0lK&Zsla=_G4?> zg5h~$fQA!ch1y4dK;Z}O@E1jO5m+hW0cY1PK=J-rz%@si6FSZY(&tMgQEv#5fB_yz zGq~6$6X5v*@?z0E1GukaUJQw)mc z`OA|BKM@AKtDico>i*8WpomJleZ3f3c;nB}1Zibs5t!e!4`A6}e*x`@$TR+a1i^pd z47UD_hqh<#!qE&Ej_%SnfMjG^(t@QIBnPn-G~RnVl_MsCIZh4`=``#=)EX9Q2vrXF z{5qvVb=^CZ?AEtYYgK@{*mppF@LGuMcY{bEbb1Y|VeasM*83<^8{NBKq-7~0_4x^i z!YK}`0xZsfjRVZRu_xjCiVHalgH;u%Y3_QVCe;|yB%S6In9>+KjFHeep_G;gxCB0C z^}fFgWa&D2W6#HcZdbkh=huR94t6oMaVPj?5085Gk2;y)!-ftz5v;C)rQ`ltdS>{y z!(l|ArTXeBs84ZgH>dBC@}n4@;74uCilI^+9if0#-*tBfgYCK#6lmD@N2*Xpy+?ZM z=s`6R{YUs0=2W)3UM4mXKj8V{hq>aq;2WN)r6*)k$j`Be3> z<6`q!5!Y}DvTa5B1CBvY%LB;M&3;NqBT8jYtXm6{Ey&%!!q^3T0Y%Ev)#`oMevwRB z`CYRfPuk7JhGBi(JpM%OXOTEZ=t|7JFSuy4pkjCBY zBePDoy=Bcvy5nlwGAgV&hTbcVMXx>V>~Lsp6#Pj!5aNG01`In93LWaK&cS^)R6PJo zgH`>W=NHwPcQ7VnMPO~&|Evs%1&fl;_v?5T1t-0`1^rOxlXV8LWbZlzBC!5fe`p8( zcc!dhW2hS6un0Pv)Ox9^iHx2uc5aybE|+8{>ar6yq_FSx1IP{nT~P@NWm^HFWI{yg zxy2d-UeBXu_bAv5G^BvpQ~sH~Wp~rjpt#lv=39AG7QG2F0>!IQXLEpA?mj>^g$xn{ zi~y?~In&Y4_pXUZQalp+Hw?zK{6{8+zvdj$5$al9x;s3*^r*^-UbW`yLhl<`!7eda ziQO|QRmTxJop%>IpH{UV)@Wd7yLH8m>c5huI~=x4i*Is}8Oa{orvI5%yHU;cr5dgx z#uYd<=&AgV;0>dIO_@57eZVQu@^xI(Q=MK(4Arpj0P5QJib2cI!%V`I3y3KH_noI> z)8;4MS{6E{#V?-WFOG4>KlbclXl9P;6s^g6R$pm#fE+0Sq7m@8tNz1keK_O|RnGIQ zi`?8Av`XYdea@N<^7f*}3AgReatvNmuR(DF;I8!gH!JpBp}^A9Ax(@P4~zc^9Kn|h z=ir2NN67dBHOjXA=WDAUTa{biJd*czOGL@GR?o9$aJwYPGk}#mB#`PBt`)1X(SLzg zhCV^QY9#;x!qoC=A@=MUpvk5nRlGq*1m(JZrgVGKdJbF%)@B>sEW7_R^IgY&Xh{lB zCniD#_5ek+kBkdjVk|E`+@3xoh~dU8y?4Yr0jmScpRBMvS4H}}E<*ULj}qy)zB`GX z8MUnTNp!XjCkuBYAw|*uON2>MBbMLufHuzo@#_;u~v+`(w z46xYU91vm5KM{xNJ3ipBR2kD-h%zTcjPvFS^LSf67W6m2*pIjAtt^Kj>DKt*;vJ-4 zYL!0w7m_DSq?BYWraF>Ysr;nda-5545$$ncI+VQS`?xRlrai~al-w2NR8rqxLJ%_B zS|L$IT`o8G`L*R1O<7wP`0h0@1^E+vuAQ~NrjEVfAj$jen8}(^fk3AOGr`&>n#5u~ zi>lnsnDcYZZ)I}QFCtR~eH*P}sf$yBjD9z~X5Pe}DhFlvC#LGYZ207GO3TLxXaUjZ ze@&|6d$HoCaz@+DiHTcALKfmAz~J(uY^`$)9DUEx2cBA#-KN!pRd1h7O~Fa09nlXn10gc%NVMILPF?u3UyO zQu)f0@~E1U@-MRIH#m((c*cRU2dCGKA)@|ZBU4LeGiwM zw^-Ygb8Pm~;}_8w3=%d1)= zOs3a8=;y8$qAh5XbECPj#C@~-o??AVU+4GxD8@@#f1>T3j9`}cwh08~m6$}tRAi*d zB&L<_RY%qhWK)vAMRp&gVf#whc1pOykNi-h?~@imiu}6T_8cKL=KtDz6QHKD?_D&A zHZ8W&EvT*1&}}OaK~Pa-3{Gf^3_=t|2*Cjn8G>LyAORdvP_QKs<_HQ%i$n$!nGzfj zNr0#*0RjYt5FvzMAcPQ-+#SH7f4_U{R=ukCs$SJUWtE!BIeVYI*Is+A@B7w1CK~Op z6l5e5ttj{E$eD^ZhGnJvu4p66IaQC7P;JB-h8G+}w&vQW6_ONb91A)UOP2d!9# zR&?>k1D!p<6kyjAoKLV^QrWz6^>;-Y=U&Nls^#@{PgO)!DKQ|l1{rp6g>*AKEe3>7 z#ae|YgX5=#Lsw*CNG6@JnB$x0rgWmcyEz9hiUt>CKl>HZzRgZD2wlX3%2O3?%j4q; zE||pM4o2>)$qMswP9y5q<6EDtusa(OFeSBwa|*lg=)y*4=o`F$hQj%}jrdPqD$7v+ zuJ|E0FQd&17VS({v%fDuwmhsnMAxdZ1DBDzL(L#a~rzF{isb8-)pH;%k}9Kp?#2aLeV6LurSD z`wkKW>Ou&sH5xW03dbYem0loL6A!OAC3I!)*0Z_}vM^U1A*bwDPp6?0eUe7XYxoVX zpKx?~k;v!oCmU?VDG78a2TMk-xDOw28jMID6pGG`<~ELI-ph&xOo-*=ot&T9XzGeX z3t>DY${(Rl)Y$VW<8n?zwawk_%%|!JiK==-yur>MPha1BZB45>d9Y+ijxNzhY1fCLvan@Y^caDmDn7Zc+Ws|% zkE`!1^R{*SD9GIDwIY*#?@0F2P+HF}&CgPMHk@**)lVqev2fQqaaE0%+=JQIcF2gh zXjp`8KC#J%7G2xT3H%AM7kglr`!JDNASl&|#Jr3biZYhikVIlTNez*F?2~k-()uwS zhtp;0ZWrMx!UEMD5YnG&zxy0XO935*n45M;deG`8u0#O*Pm5ESZ}d;$fgW^vNL2)~ zvfB6$X8nhdqvzcZ7#F56IiEaBYFcuof74U1@G1A+Wj#C3zBgGl--dgXnfwZ6IvBI5 zwAZU7%H}bZLKMA_P`KL>JCVhkE>Hzcs+=+JfWVN|fo0y%6}W{{AO`_a z8C9`o_wET@fs#IXbQSl34*!+?Dn#^h%W8=CYAsHKsoCS22i{#mqzw8rVR;*;Mr#0#-Wir1w6 z*lYE5Dof$aJ4Thx?wKVwmT|y@SQ*?yxkgJcCCtF#UWi?UZUqRcTPN7pX2MzEma{=5 zQx06XZK2LEVPlM__L((gBglP4@%6hn{Wt>sZG^=y6ruydsl%8)int;s?MC89>4SZ{ zT3h072Ri?4A%1KmG>I&@z^oTgq*cwL9M@ql2ctrK{zxP?P98W!dYAth{q``95P9H5 zm7T=8ZI=Re5NBdSZN3BrGYE+d?TPtwCI|wXW`&j#e|E*`I(r;&7Al}Wi1UqTOpdT* z-q{kO|7OhL54A=c@%e)K7k*rx(;MQVG10hGNNKUyfGK`e>r@)Osr0_h!69QsVDS?+ zc>~LU2FAj%jZB>E!3(!9wu?{#Ki*O)YPT{K0aKEg10ML%qIlu+eHNI~V^1yYgcRt2 zPh-snGX6?m$wEGdG%X>>Fnm95rbD`k#?t$RkMUQA9Q8YrmQ_lZn0>0o*!W;eceqfD zpQDcAbl2bHvfCPs^=U*8+%^|4WxN!rba?YZBS-H@ra|~8i2h9<75@p&^z7wPXS{!NeK0!j> zC)QRDw)k4tHjd4ek9M0@*3>-Ag$`9x>|7jBimZESU81cWBLQ&rW?t$bJr9FL4hh5A zJ5;dHMV~8e`Qx%OZ=3vB+bt^Cv%lL(B0W0Gf9x8tuppU5V9NL+_r)*cM>&0)ea>Jb z+Ae?ySs7kH@&S|+D2c{D{T*uMv9fK`x#exzmycaB&-}i>{|r6Dz)k`VpsqlR6IYmR zVwRQ$%lAe)#WChf%FJyKnwyY|jMj)9bBXkkgstj)^qa~DTM%C1ZKsEx&fmqXZBbF@ zSQ=X{S$;(OfD6S4m*jYcv1+_qVBL1YKN$@*ISd;(VJDe_T&p^5d#(ed#xquQ8Y}DO z97a61Ekl|n4?O(q`L!dV>oYM(vQPv;`-B} zT&(6ty5A^Aq2ajNgxU@q!{BTD}wPzt|+}6#WNG+f+d{kmi{}W-8 zLXnH>Rlvf&^hXHTu0Khnq=N*79E883o;~bN9QpBm+6ad!{tdF3BgyQakQx`xjN?O% zb=)}Y07-63lC#Ev&&G~Mkz|aoAPEfxJM5RP` zhqR?gr#uuHx{e#L%Ok_t-6nt7lVse;Wx$#L)MC^-BxJ3E;up5vwNm+Q&ZJD1#J9$N|FoPCj?0oF%(SS z_4hTx4&V;s)MAg)n%G8%QB|}S=uTf@&7nMzGS^TSzAjhn0hq+>+66MVLIVZe!|`2? zC3rc(i|mlGy!C0$l@F@|C}~Ifi3!c{Hd_w6N6Nls+#Ts zi}|g@eut8MV0!zCf2yt}^>c-gKajES&ij-ZrB*}eRv|I`1A15OlT6)IGJUrE6zm+m}K^U{;m2Zlo100@K_}v z$2Z*TJ5+Ogd@xD~a1)E<*#GEQnz3dUc>Q#i{v52>Ug(fWJ?$ozCcHh&w)M-favQ3+ zv8+Gz&I=|R5$%(I=Y-;B*(??VzzalyFb%w^mAh{i5Q8k)aW&$nWrZdHg~AOze0P?< zL*v8(iMIKN(yJG~a~XA8lmJDQlMnCGj}UW>Ijkl4FD1;rO==*ue|HtDU8wz|iy*=6 zR@`FocdMlDs*~UFC6GXVz3OzCbI577kKS$H&6O#Z7|gran#wu0%Bp?G4xOOyXD-cH_b5i`X|RP`CY~;UWEszm zl=(Sjp35Q&HeZ4X1#W}ECy`^XgHx4DT-Fqv@crYS(!oYWI3ELNX2#87nA(?07jn|A zQ3Ou(O}BBsVC|6DzJghZ6w73Q5fT($w9brvBPQ2_szhxvAzm z8%<-rdp13T!EAt->wFoaVkPc4W#0*#Ds(3Z{VMPr;IbhY}J-a?hSV{)pD^R)HmObeu8e_2OJgbAdLgneTm zFZIfT&fmXCK8~c8e+whF*d4+czONY->t>b^~#{M1FcAu-PpPPp(d2U21gD@s8 zB$*uAr<0g4XKWy9D#+RTfGkq8D#RQG1sy-kUDI1Z7b4b4;8s6FoKn91`0C~z@|W}P zecY9kwOIn#!%zSZJ)g6XSg}*-fs(Zv4US)*-F%nickc1Z$nL|bsAT%RgQ$AQoy@tx z7)^$+PmNJJ2zz00*MvJxfmhH?V5hQ)QeMnw*(WARG2t6C+MJ~5r^?irl7tx2)TEd? z#g==g^1Zg5AZJQb`2l1J*H$XEl~2S>uGS?hOxW@dM|rJY&aOcX;5-u1$;Ct5zvQ+t z_^k(fmMbG@NUzb!GtZ)dk>x?O4wXsF))60@>{Z`ouRWQj(Im%p5$0g$(Lah~aOn5ESK~SpOm5xn%X|M#j&BebB^Z* z&ycR$2}vAY#0S8yU|4)vvm+)n&rXudo1&9r&Lhl-`^O}qNcZSg{)l~iXTm>gZQUk~ zO$cSu*HbrAZiHeho2!1!xh*syR3c0hxYLjhD3wm+4lkr$NTk_F10taf>(sWDBQ z!l{mLPuQ@G6Z83FICH&;vpWG@aF$s=n)@-DsWr73ykO2XG$N+jHO7%tWL(8;R9TF@ zBAJ$bT%Ih1|HNSWtaf!17=|YW_PyRR72?<_jA4s7k`8e@lR@`9EW-pP^K=bknZ8!{ zO+}JU^v^FpKEI#_U|g#q0l(y&%}cB})J6NoCK38Rh0TKXU6`RogDYHPm4O%hnyz89 zuh111ycxSN7M?d}GsXH+O`hq&kx;_1plW{rRkz+aCdW|c{f-)rg;i+CW7RBubjnJ2-OiE#_Zc}^Zk-__MR><~xS zCS60U*v5JJ`*+i&{Gq6z{;CF@!cyrwF=kbDB051>DKFx&!-KcWrhkoEqEzYthWM|A zazH}KP9KBmZr*U1s0%_j=07UI!*jc5tmyrvZLS9y6%uT~{+4U*0%{Y*DQRieC;4hS z2@ls=Q%!r3urfi&e<2i31bGZl0KsPWvFyC3uEBv71o${A;RV#UaJTK26d1lpm+ByG zVXyP~yAq=vl3mFU3lu8Q>DB(vA|lc~*)MqhEJV^OnJ@qeC{}6J8pP?EfzeXme;PlPa(ae zuaBO8t>-Z@m?>Qt8YK$*byP*4qS)gif2k#``eYOC0xQzLRydvcEGWJ{(;nWG8dtr{8{bu?<7i%O@<{yqvLY`o+i`eT2AB4l^d6poKn(9+pxL4+XCD zAXB8pGoqG91xn)a?==`^Q3CItH`vn5&E}$+x>IHBM-Pms5Bx^J!&7})(gy#&pvjIY zZ^)Yt+#K!_U3`t2R%t-Eh(e^AB!{5v-$JJ+QcAf-Gyp#GhzzreWD>le;7W3K!F1;TF!&*2-4_Crr z&O;#)m1fv0G9iWR`_mDwh39fngAOZ~-o{2%XZ)_jCIGyif*C!hWE72PK#Kw>uZAzH z|4BCtjNmkgK|0J(+k9UPcRCL?;e&sWqO>?nV_M;hYNx6A5$MIj1D?xoNN>MIsrO?! zpIOT}^nKl#7%W0QlF?5A3>*V}>Y@e8_fu5^?Y&Q9tygQv!>j5eJZqDYeyOO-@Pq(piIPG5{O2GD?6pkgawJeGOIX zHhqZ=g9;xQ5iYD|RnSP9)!>6M$rC3q zWE2j2IA;kmVzI0|O#F+43%8@}Kkix}w{dVeuw~L?+Ap zV6d(Sx&FmM-Yym;?L*U&U1D@e-t_Csa&+M*6VmxMfChl62T|g&FV~l6D53}S>`^i8v@Vbp*fUi!7+d3;5{}+XgiEMk=(Pxh}%D z>arkr^1hCXch`&t==m-$W?^XBCZgL2C6WL{6&=?5dq@`&!PPsPg_#t`Io?s5MD4z^ce4|!q z(s&&dWrP{_AP*xE?cz*KZSo*b+op!B!MZxVM&8gqunEmyQe=P(+E0pT zGaeg&H>^KE%04%C;VT!_G+>a{RBLJ?N2%H~4}*@MvfY!e6=@|LRVue{D+qlg-o74| zLp2}Uh&>~C;9I{kTDK;uCtQn#5?M1k49ba*uyS`Fb8vb+*Aquv#DtzHyiv{1!$cqu zSA?a5?Fb}!AJ{A!!?6f=vk|Ln-n0ovsJ=-gZ4gaU{oyt^ayVx_A&{{7DK$E$yWpT# zkS|RfcNCm5SY3Kay{x2?jYx8iu-WAgFWDba8L}E)r7P{sy#yT^RESmqou1NS5?MHn z5Yy}H@tJ)Gn<1}V)6ht*bQ`^qggoo*6I(&3JVPQIli5a>55vp1Inb!poz$|VrnBsj zfu|))qvaHA)3B+f(aLBSSOfeVm5(0Sj#EM+wzq;+7C)blCn4(?USk=bA1#Z~^nObrUXgm(!Kk&GIOxZuwB)aH< z0+h(|(0mVtuv8FfY&`eCNZ2Qu+G+mD`=v!blHcFO9fx$a{gD#mRP?H!qTMqiuxt=c z3tQ%EvltiR#;3w*b`rF}W5NV$)CQE_bXA?sj}t!11j)JZ* z8+Ta!NZG9(K+$W6I)6%eMCUbeI;FQ*jIeh;@)B;_AE51)A8$!-HCTv?ynvjeL4C+Q zGi(Y|^1usU@=Ts*`6w5AgVsRmN(_;Wokw>&F{et(<+BSx%~;o!1bDuSHqsMrZKn>N@V zT8Ma1c7W($%NAE}oj)M&9t7hAPkt7Fc+>-UHB_YJ$W?WjXv)Dcz6k;U7^hbr9(dSe zy`XBiU1eV2XKGGBPYSM3YRn4X6I7Tu3;&s1-GEe>*yL$bZ!oOg9U7|mo%?f#W zp~}d1{MYerH+l51nm}OtJ7MyJC`6=JxaV*N z$SY#I{r1<-;Cjn%!UIOuSFz2Yri{|&%v6fB1#`F{$2;d_;E?M4+i+Ig-;EzcSnG-g z|HRF;SZ7Ezy2MTrcKbZ-gQ#K*v4X>LJxe-Kd@dXO6>JVev5P?zZW0)_0)9)kYSYHDW(eeu5BqlSI9Ll9jhoFx9`$x>Cr+ny!?bzZbCmme z6i(wF(e_6~i5Jl$gE4MqsFcmN=Koy7*w?d7FOn*i_1t}Sup{Z9&Zpq9J4x4lHyy;O zwFANX9M=% zM=p2v$&I#8my^hNLF>sxflF-Yov+AX@Puo0Ow7>uU~8Q)*B6#HuA-E##@&AaQNTZ7 zCdKT^0xz+1gyKMCjJO(I9XKO6u5Pxk%Ni2*1riIjCvPcG8aiih{a822SVQ~5?LmA6 zK!4{>-a^n=muHbU4*$A#SD+Dv zbWUCidJlxR%}D$*AR$HQT4&q<)?QcO>cIB-MK0@jDOiMUL42mqZIK> z|M0?6C1SZ$5aAmMX}R#N7yh)kYk_{SokL_A?AyCTD^s*5-$~+HoB|mw72ROABz^ zmH?TQ&FZ}$e^~$&%yXnVz?m`knc@$3XYS?f>cqBFfwiWxKKexKR_7ccYH4IWQtDy> z|0wWM6lh%T0vv4qmnTO8MZaXIaH~@I3~t5R_#f4%^ph3Vr1<@Bz){>6_4K-7`?eeUe8f@ zqB8d`HkcDggTV>@-4)~f_sL{9f_sUM%^m>*W{%ypVDcob|HE{NX#=UBw5(Sp%#c{< z1E!xFPFXDQVmAEXaQ-Wd+jLhI_Z4o3m88(Pz?v^wdVf=q0)iJ;__a)+r@wQ z#ZB%0cYWUNdOl`)`hB~!Z0D7(MJ8N0k7M;d>Z7WSeBWn5_Oa>n1uiPRT0U%e4{xd`w4Av9ilhX4FkV<^`E6EvBX1wJ zTGx68SXZL!Bi1PudDUfVP2}feo58_aN3qe%$y1Q5yBb#{aaaKX`j%@`ZltUR;~n_Q zoWyoOrGRY2fh5BFJ~%ihL{vaL2*8D( zWEkSty#P`^I90bf2LG{jh8zPW1+80s!96;mJxFlNl=`SHohyhATQ${j zCY8YN=_btsY94=vSFx9kzncOnGreZ2H4pL48oar}51fM50s2xmj~d}I14!_C+dsPt zfJ?5FTaXW_==45n`s}qSJl$s`(DJhM+-Wf{YoxVGdcoAi zZ)~oJwO+Z#vI9O&5Ao|mL`VI=K#*}C!V$xgBhQdtwUY;$sO(fx_D!|kie_1ku#l)2 zZnLOx6=;hGoud`b&Lc2cD_h~KA5L%*&GeC4So6Gl#Kz^DPGX6x)s?Lai0Fh<>29b6 zDzT0+Kcx3XhK%+YtB)@EdH(ODLuSi=`MY~>WiUm&`-OyJ}T$|%Uo7vt`!_ zfQ>@7-TNd5MHIDY3shy~SS~JH5jjsr$0Ls*Ac@UNEGw|cTHv?ON(ClBg?8-`qd?5ASIe zdYLtlaO6UW281_TRmL5N%KK)RkFEx$a9N&6E>N76*pib6C)|FEMYA6SdBh^QSpUSp z7HKwEECI8mOirfY`nNE#ZbN=Me)I8yZqkA&Ai?72ru?j8o-NJ_T+v&*SEcT(%ZzXQ zkM``KrX-bxGwU$=L$5%FKlO;+b9Lnd0g-Uf6#C9tQ|AFvl_$kOdhg;i;jRwHHr3bTeuY2E>Z>9s`Pn73>2CDCm1jh4*40GqSQXe3&aoMe`9t1axEiWB(YRWqoVe9f_e#?%i zfuxCHU8Z}97d6JQCZhKZ?p_6~fdsY#$O)3UR$bXW4yC_9%Le|MlZRBB$s8q)Di92^3eAsO~pCCFoq9)+dw~z*a&G|SxxVQ7Z zF{Of#b_e=IaQx_BdM3yfgZKwb&cVne_1F0u${1%65w#uMihH<`gz^yU@lG{Jz&DZ_ zJjMY(ePdf?`IW~>R6=$f?2rT`1fPF=_Ms*bCz-0I6Q>GD3fy~EV9xDrME`2*9}URd z8>=97Ku{;qrz#5L3)ZaB|5*TbQ|0H8>b7_y*xIyqn{J38*hyCV$Uy|`QP<5*J&Gw@NN+OQEUSbUJ@G=vP;TT< zRC#3kSxB9%<`mX*zwdnr#3}Z8(VWm#+1p*|bzRFN;ZE5P&Qr)sQ7$DWrVUuy4z=sP z&4WdYM)()j`2@j;FExHh=T+i2OJEej=4=Xu-Laevk`*4@=!?1l+{Wg67mUefnR2Cms}4B_Da!m{W8h7hK=QeNPFn+>6yo$Yxu6ohf=GYYDy(*0UJdUeoTz>o z6l&Blk}KV6A!utT2azT`|Hv0bjq)N8ZMpvIF5nxd`0slPzNp*h^8J{CZ_J_JTLJh2 zqHp(o*TEOP_Y6|MX%>|~$If?5URRdrIN^Avpc zlyl7VYDFjeuVar*6)jYeK>h^tdV12rYr%KeicqiwHM& z_QPiL7-l}KAUFVnY8R?wY?yH zz~-POve;!U!z@3Ra$kK292HR#LnP-?TC_&13*K`DL$@&$5~>U$N zm&&#OHQ-MV;0Q75;*Yy3tNlZy?}Smv!J4d{1XgCqYJ`ud%9HBkLUBZ#b{?cs94y{P zXAh31KcOB8eH$I+UH1-6h`buh^4i;`Fk^xDkDRb-t7IfB1~fBa$(aK)?EdU9wOiRq zB5u!{JbvdIjKhOpbpFFy!9IjDp$|7&at1>>vG!x3TAa}AgbRZbkqU&DT2lU7g2n== z7B|jb#P^~1M;F|;J=oR4eC)S>aHa@oF*aEqYRlrCCY#3K6-12#uTEHVfJ#xVk}?&b zT1m0q+h+0OT?E05kABCy&D|{(a3cvx2bnZ!H zNO}nuQ5(b|0G8r=gCge|#7N&Izh`Fa**@;4v9M_5E_%8Y%Oj1Xt9=nED|^xVgPlZ@AY?tcSv8IM5s+t75$@AH6Hirr&SFnrj6uMN z`}-39x!)ab!@-HP-pA=Eaz%zF_)EG|ZRM1kx%=dz&Q$-_@YrXst3Y8#LtBNQH#Xp~ z`$Yjb)~0b*5;6O*O*>uWqlLAoqbJ!fMG6JKyEakVPV0x)kxSOtF0uG!joKR5oxlIS z{rCR3?JrV4$Zi-Pf6e))sw$Ec^(!;yPwef_6BEuNqf89?H@g9Y0&ZTIAgZLx`T(Q$ za7hci+5fW@!S|pOgQv+|-R-=g4z+BAcUctXJx}3Q1nX%YuG;(5X2j+;Gni;VpbKiR zPuz7}e79LEDpU3Q!SK$=o36Bzr27dTI=mUlU(V14&ZX#Q9VX3g%qwpcK_5z5Ph%9< z-=FN(TQqW)TZCNgctp3h0h5^435zW7yvE>K^o5ok#xamd&*VK4;Xtglj9S9#8GwAh zD$)uJV&s>{4C93n@nVOSiW_;;UdJ=e=ltxY&)}XQCq}(EC&OT}kM`)%2g~gwwF`M3 zVlPI_+9YFI!^>-LnEuC`qZ*Kv={sl2IOc2#a-O5bLFT0W-m?Vv>~3Ax{+uqa{Vk$Z zhnI8iUSn5N({ywi>c?&t6tI`(e-1nALRrp6ZmyU0H2=I;uL}tXa?bQ!|4?TanDaFP zl!nT7{@2ZWH^&RPYiS%l;Tg&yCT8&Dl-Gxt$8P#$JOjBfUN0zurY zkCJcbz03y=rh`%$LPhfiHgHPauiB|LHJu6uMCm&BO(n_=J6>q2+{ zRhA5XZOU_pi#(%9=+Z0aH?Rdc{k4a-CytS}bYQ~lqhR4{?LQq}!9i($@K_(jDA}it z7i?q0jW6w;Xh~1H={N3yTpehgwt*&CCf>Vld^$+1_qd@p$R-@dI2|ZFD;Wa~69uQn zR~Bck8A_JTnbRF|`Q$FqGnS0<+R#yZr3vHaw2y8;VT3Ot6 zIYp_8F;i469rMu5Y+-*?*LrzF1ghahb$evbbWWuB?SIc{&>3j04DOJ$@Pms}qAPlB zVgi+Zr7ty}eHYqYiOHhstGBphP!NC0t(+I5F6|Q^UdD;aAAXZD-gcMVaVNm zyZ}cVl(QR-eY9}OWXW(X<49&YW3M)x;j-b+4jHp6}$_Ms^Nk%kX_hf^k4YH=_J)Y@>G1fdcgx~O?=AtG!+b_zCri-d+?H}r10jN z7LRqJ$N*W>FqYla*;LbKu2&LXJ<_x1VpPed>Y(rUrQmDdXL!Ki>Ri7eRkQY&`bB{E zWg$G$`WF`gg5CVNV{5+r0%BzMRXIHSoj3FwmN)y&{@D!v+e$tX|No#XYZt(O`5XO_ ztmHLjKilWdxB2pa$dMcW8T0>FT9Epm!TO{Bis$qHGvqh=&#VAE;&pv#v}|#35n*25IrvFg1+|Tk&e;5izr$^N;_3HRb;=S)cs|8g+~B3LC;7hqaC< z1T`^IZU|$1gs%&Wi7uFSfPS|T7|^I~u0&r~oVcVdMvwa8%Rqv69`T0DKAHBkmg5|4 zl6{4yoH6k$U*W&T8W))oY12AIu(btMmS3;@pMU(%2>j0o{GUg_nQyYXvHRU-`QDj$ OY@eOe?vkC}7ylOrOz1lR literal 0 HcmV?d00001 diff --git a/docs/images/DataConnectors.png b/docs/images/DataConnectors.png deleted file mode 100644 index 57ac81bf609e1fe09624d2dbdd5c2277dbd36a6b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 47323 zcmeFYXH-*L+b9})FDQy2!nOgSN-t6@6eSc1y@N>aT}puM1}jA+1gTLlK?QFqDKMB?JiNuCT@ZzT-XP-tXM;-FyCg`J-#DXFk2oHF@zsOYOu_j-wzD=*0bd z%1{vK2m}P$%Q$=xSZOm*6$gRBK=+kz>pdQr8CH63W*j0ldXy)zHCq1D-KOZCljkyTUz`mKcTdTpI}0zmtJe*dF^|6>}sF|xEBMAK6K zAvLCL^@|Iu0#+$yWn*TfFnfbOF!~YEf;>^Edz(cR32_E>hRpPd>x-uN~>nV(f5Ax#qZ@lgMZNV%$ zh`qnwCcYQEegzLi^3Lzp+5aKRRD|Y({}bw!`BB_W2wiCQ58W&em41f|Su61u+4_zg z*&O$;gVB>n+aH=1n+ozn{Yy?7$0$`VO@fg_8}7!)+^DzF|9F?i$!fv4&jiLZ8M)Af z_)uTn!U20ve!BiqQ-)cRVcpi;#9Fj*JUp{l7%x{ITO+;5wKf^Lt!)eb?UAEWmvoIO zy!5(N8GWbXkDPEXydsy)94|cqR~o}$%!Y2xw|>pSP7a$R#cZ>;Xu)-RyFzr;T^WJe z{J)+2xD2AW`$5MwT)V9klmth?U;1rcfOtA`+oD-hppPkvxS{#>uW1_$Y=NQGfaPMM zP|zuX-@9j;isDgKYw_jVo9QpqEfrx!@#xzOzRToFALQUW$!P(jS;n_|M64J|*;xH| zoXH%|)I)IaYn;%uP!CyRC=F?3;9+r49oaTvdcVJmIhME1AbJ!6P2}Hp>tM~*NTYDY z8b-I88d%}I-{l&?IZR?z&XzCA-~)dIE!He2;{c(#)}A(qAhbwtBQ ziMg`+PBULWR=w8ak&Xka#b$Tw7uc64#|}~y%Y$K-ECSmhI>y-+<6;ZWow|Q;EOGJa zCQ=4d1i{%T2ckK0alL0g)@_>qK=s`IqV=un12seLr1khFeb_wG))tNRh5ieR*?cb0dT6pF< zZ7S&zTyjn^9y@8mEz~Rh2Cbn^LA;`*8Jmybdo>UmF?btOsXqlRyUdf^9Nk)2p4p6i zp>-FoD^hSBqN4XF@M5w)=0!T^!iIJ@cUd{L^iMB2?gR@uaLl% zyuke=eq9vhOj~fPCc5R1#8#w;V2Ce7_EpA)AphrX-3Ux&BTYHY zmS8RRUUgb zDJ0foo1$_(Q-y9Ek2-&@<>?4Fts8Ain4YX#wVj{!;cjjZ&(S1(8D?Bq;+n^1vc>ga zV0!7l%aO+!6ZxkF!eBDS+j97dx}{W#U>u|AD{^j$(p-F%=%+y zlSCJ^VBFpZ^lufmX!o2D;eeSzlkvGXk8Ow*rapM(j3V8NMg+FKdb@g)G?~)zs z3wZb(ok)J;X3Ts2+nDI@H$N74 zW1>Hr`nsMZUr_v=$;0+!8X@pL@9$>{>Ta>)^TDE(qL9Jix zJ}{&w1|{qprD^VGS}H{@YqFu_Hy6`?_R;TI9?4}q{vBf=RhhVAy_Z76XWAZplE|DZ z$jlYXcqcXuKID6hAAGgPHWBW_a}_T|yO#T*Q^VW^AD*kym972>ZCXq*jh!nvm4|!x zknkv~SP%Abw8|C@|MB#t+bcI-YOHU|QWL$~d8tbz6yxJcd168LweSmmXbv~k8!?rD zY5cA$R^c|n5Y!)U_d@7KwQf(38VnD%(uvnI;FeIU`&H%UBnyRrtSwrVhdy@+f+1v^ zye@k@?tN7ZtW(xN#JK~;RXM-GaAUFW)a9oLFmf4^v66rzB<7hSvLUK8=h!sprrad5 z8kLQu-!XrXgY-Y@Ib3XPK~Pbt!1k6^a|P5c4&((M|0itf596Q+lk4CD!Nhi5?cl^< zzRGLEUOGqleT{iy(5COunNphBVvtQv{Qh(|*=)E^RYQt=bntw!S1bRtfJ|JsZQi0x z2@kD+jAXdk6ssC%3%>@qq1<#Tu*fXVs3fe*(lifVl#4DJ(lS?WR_ZMz^P3(uc|K>VB#Dz(dIG3#aTKU-$_GcSoLo(d zSJ=TRfjcyX@;P{sN{sgu->rYVo8@GsVB8g``i)o645d78#u^rG1r-LLeYWlNJyiw%t*B)F~&Ygw{qF*wy1A2$Yvk~Sw9 zRfNnTV(}dvy00;2Zn(F=g8ISIba~5Ynd=@4I@r}eYEe$GItE(?F0e695-r!M+qz5A zC1wP12f&s#Us*Kk6E%PPjRWy=JWBaT=|8>s5dZ(Q-#~-?KZC~3y!HRrpmE3C;xtRY z;l`2Uf0G5x#Qon6q(?9Q%Qy+z2e`5brQ`QEb`Ty?>|gqOc%Py?S$uZqahfOD*u!Lw7_?EG^%epAv*)hxi~ zWvVIER#cdV)Vfo;m!>!5wnr%cY`?5F+oOWkc$-}(tTx|cnR(+M$`J+F%9=X$J=q0D zsDD{5)qe$5A-#dEVKYnmFXikW5r8=1yduKf-4U7gC2Mq>W7@C?bkclXMTN8PF14_~ zZW#HPc#7%GhokBDS(^EOtGs;T`BW8T`o?d_)uMnW8;ez!sk|;l5;c z%9{WDE|x;xQ`~KeO2}5id2So&&?HA)+)52XGZF9Irt@UYWPblHpp1?UUKwZf)?_jO zadkaD|uoaRBheN{E^3+y9~X5dtW)m z5?dADq@3`e6wl&;mtoL8e8W8S#CyVafh2M{mk>5fjR-fqjyrh9=+j{N;{HNPg`$e| zE{K)^&oSH2?Q-Nqjnv{N&jcDWS>GFg>*CP5_}}W}tcAWeCO_U0UK)o<_`|mpU-UAE zvo}+I*E01TQ;2i}p{7Sy-aKh>G?}cxVi&vFO8V^$&iLqvefO%52 z!S0D**lljSj=d!zeF@xsl2{=(ApL=d+> zrVdxg`!0>t2FB81Zp}rncU8j6gB$hDR8>f7t+On%dOP35VlRC*kX1tl1lA^R0-&Jh%B8SYbZd&fo8s%;Y(IAylFUYcIO0aF7DYVlP9I8b(kZ=$ zn}P8?aNS_=9cW-Zr)jmD=K5}BwihiZCu<+Q#4}bO3%x&kSHNqXs-bH8|O|L7WbL+(IiiN*&gHf$ITiAp-3 zMLDAnma)rD?%6F*(OS_f{A$yv3uSrZwO(fL?i?7x=5fXMFEJEC-S~8NwS5R>`V#*& zK@*}Q9T0jEtPU_RSdRDLggWt`%?f$P{k|{MMR)zt z?RLm5BzJc_R8L)FFoo5Z)~I)GR8Q}EcF6{N{8;R7O>@2mru0FjArVv-XEV@q!^jni zIVV@W4PNZlKEH>(I8`!6aiefcXYMLJj#0V@`r>Dtwm2#FiyK}M-K52R3mc;fUC9{p zmlUlV3t#-hhO;1ggYpOUcM*l9FZoDS>&B;H0(>k&Zp<6;hJ9g_Y%eq$(iiQQ)_k@C z*{)}Mxi|qQnf5)uY}c1$zGYPdPxGn{xp8uI{w0g<0VS<;{I$CRr~am0Z{ip5qw|!~ z)K*vN>!Elk>#cpE@`Qxx(`nAt3Kofx+eBl)Dzx>ry=*^^DIr$CqudaoAGB{||RvZMEA1I5%P zpEYsENEK5+Lr*m#_(LaW(+t6Hvd*Y7!RB~dSW$s_)-cz6Ocaxhu>H4>5=;@h&_D# zU_KVIUc6_s6uBzJlZJ~HK^|3AxOH-$!f*URWz_T~XrWi|Nl@WE(hv3DFPtm$(w8_R z_JLfo-3Q2+?VF9ediMzDOgRK9{kr?y4pN)k13K=#OP7?H6ELU`_y zQETW30D)Xg6`~}zQ>J%~x$>F0%K1|4lZ4vrZxUs)a;HFHHpG-blW5xDN+@C1Jx0ID zFyA0{sf>PQLV^&x!ql^P`f!0jGmU^ShQfCW*LQfk)y32R=$SPMw~opp|DrwmgNX|i zJ&28_QMdeIvP43|$sPFm+g?m8rMO=ztH1Bfd@hXqWy!Gg?@ZaO3^}O)THrU7xKTLz zT0zMR*=fnUay&3)WKJ2RDOC6<+KdhH* zgm$>(Or#jqiT|?et$p}YhrA9f1O3kcQ&G2;kg+QPxg?h@KDM}DRj(dl`(^vf!r2%p zg}}6GbO9@1K8dADr)Riz-)Md%@i%NcJQ}>tc`H7Q=HxBv&~NWr^Uco?R#P=H2RPym zJvi7@6f#V(5!Ty@>aeD3h8N?nUh%SB;%UfonTStcHfsV#sKQEDZmJ>A=EPoh9DN7U zV-eqjS8^#X2t|3(pFeXc7ElG&XBJA#C$p9{UJv{zNqJ?lLtf@crMgVgBv{q8IAEl9 zU8a1JVgu0rvb4ZgH@!hq#jVO>cTzx@&1-=$aWWW{;p!5ZdBN6(69h~-N>@{N->3Hq zspZ|2M)EEq{4kyHlCwpVFj**8u=# z#!8$NohQU6GjR2jeqKQaI~Dc)ZLp+mx(qoBo-eG!;cMpWCYJfJXO?#}~}yl~}u^16@U! zjUEr!-6YlP)6N)J&+;w9L&`_|h`KCLLc)kJ(Ign+*o%?S30P^{M>1>vVr}J#oFf1U zkOxSSv!J{)W4@$+gJEWyl;c|B>2eMr;c#1V-{hSSW;fu+o*96($R$3)T8qXzv!4|~ z>^A=abX+v+yS9QQg=iK4PnZ&zjAs1+eQrYb#*F_RPJSObi^}o)(JspqL5sBpOx6&L z2dO60dr@9y$nn`7L8M+6ZL!ZIJG=T?`#B5ftK^)ZXr}7JTH(jy4P=W&kgZD!it#(i zHxgc~tFA3zUh6c&%<(fCmG8mGUL03%e1W9vPO*HimTz$B=7dsGvGm3Kqs-5?9s&lk z`GnFPQ-z)K(x>!!ZjqlWu3$Wx{h;shgjGE>l-sWZ5hQJjidoT&up`gWPK(s z5b^D_mhI9aFsj)1%oFrcVAH7n3z{GBF9Bidr@vuu^_yPMO_l?uBx0?V0OLIBeOVjV zPG{9#VHRcVZe|zj-9q2G5g5AV_g!_$CuH`W0+SGIZANM0Np?&Fdb+1PMnD39`A$M1$`SYrOG_-~~2GfUjpUJK#vLRuousc*uYN=NH%+YjPjy zOixq$dy?jqvuv-UPo*z&=XvQf2SSelbx)`M2*ZBesU(dQv7UCie8?Il_1q4Ns!dPF zI*V3iRp;j|H(SS`JsYk(da_Q;&@}on-Ow;>mRvS!JPk^xiTXH@B{?{+YJo zX`SYWF*}W%5vA!>^C4%~J8cu&gmXzuU@6p3_|;WnbrzKpb#5nk&*Ocv$_fUO+X~KQ z01#oWfs12{+qu<-3p)ogJAL+YV4$a`f9%4_{rr`dtl*uXaG@{8cGdGLdR#JLA|_Xm3GOQ2LzsJr_P`aZO)MNxl$q;5n)8c4aQ-kdVbQ=$0e|PR(Z&+M7Y6$tx5Oct=R$>POhg$_ zj{zbaJp(ghCMJ;8VuQ;{zmyc0cY}TYM(GX+<`L=PMcs^I`kTRG<^rg_V&_<9+AdvZ z1$e9%ww?b@?cbQ?3A)g~ujuXHAb`PW16?2Omelg`Oa-ibx5Ib4Y&pcD`wW=b0K@@| zo5X&zmbhf*)Mj!USh6A&^A^nk+(Z~LA3+Y?a)vh8?No6ArrQo>_-1M-1bD91?;IZ8 zN>6Q_AjKegd}$Lv{NS>gs(5xK=o}$)iP{l?rdvAqFp<2!y>qD1!t?N`ets2`m6OIi z)-ggFl^1r7@;Ko4b!;;3GtW(ioOW=y&A`!|B>S?R-Iu|)gpzn@%%o$$%pWpR39nOX zqrSiTGU1+gh6y5Z&G5m=^Qa6%QV5Vl?c#u?Fq^nz4d*p`{NC)$b4-X8bN#jDv9bJ} zrdg|@9ga2=wDV=|f4`^Ft9axOc_7F&VwuL)(#gkVsAW&Z&3rmq2Q-A-ZrLa~h0G`G1DqzA=oJnG)XH-K8VRRMheCjEcsQeRzB=+(b{;va?l z_KE-6(l41@0ow!iVqpnl7SBGHvn(u30scFethO5gP41Wvl;nEl%EN!ND6q7M^WR<_ zb|-DpNL$02i`#iG{x&f~I`8Ami)+NYyQ=iV^sfwVuao@is`S?Xe8KvZ)nlwiB!b4SeaP7YRc~Z2CP+ zSGM|9jKTA`N2H6<0a%+5cQRc+ggbDXG&1p1bo=!uW{@Nkw_5p;WwBxjftHLL)g$x- z;3Iz-o@1-Qa;vBHKsW>YpfH$gn7(P}&co&tWt);sB} zoGz6ApyhiE@qXZtAFrTqY>Ac1&wMu?gaBzw=2eDePRIE=Z@g=8B*?hNEsZ7Ni~9n} zbMSC9x%Yz&wN!uQ&q4sD$H~p4^%}7@7Ha?7=FoW5`l77u#+sP7DxFn8D*&AI%- zHHOP5xnNRM@LX-74U!NS(uDpD^^~9G5I3KMu`8qyUHan#M}pD%mLNXnjf*h5!9PDk zW$TN5Xx6YNBhEj%_Xt{pAqOYOKTSSVJ7}anB)%U$@)^1&c}5e5>eY{gxV5CFUCvP? zO|da=4Yhp;)kQiJ!W|a9oN&qJ{Cicaj~#~yxjH9b#MoIXt!S7#uPlAAhrgVxmZ?x? z;E?m>QH;5WbNKm&*@vd%o?{s2_-uiaHRT3xKKfbuO{ddknCg?D2C1_AuQ|BPmO|DT z>7!pF5#MvR^F~Hi1BUG7k$obzE_S$UA?1;G-S!g=^XASXvT+aX5XChoJhw&L$bF_9 zxZ+md(~iq#ludT}r5K$R!)}c!bcn@5bqRz*7G*a-Viw&{97Vnm{gkFLT14Au-i({P z8K@9xag3ZjRIKTI$I>&DT&9P)I_K_s7BhnNLU~XX@y=*B`NqeXRMV~63PK-torWpR zGbJ%3fa*EAvM%lX1uENK>?3@BQkZOZtqgdrvS9Zs(I*#oaIbm9@Bpin+Z%c@e zezTrji5(uW_yQ#fV(eW>f^=&N2|3Fs#kenYk75By|}Z390l6LwUc{5SU?vFJ4r1K-%;(G|%Hfr0u*PgrT@q+hfMD zsYR{)DhiKjFe^%_oq$I6B2-;A8Mf;Ti+!T0lbur07RT1sCSBXc2b|h57RT~(-2G!e z6g`d{@vJcQETeY%{a}1+zK+2Jp>Gdm*OY#77+Ne6VmkIZlh|5t0bBiZ?=%Y$8?;n|fU3bO!AmoU3C>mcA0SIRL&~oChoES8UPH_C+6xFSo%Es#Y#c*PiqWb+_}~ zvs0>tGy0{R;E{I4vUWz<=Zr2Kc;I*OiD2zh<&E5)pAM3;l-cMa6DJVuiB+{~n2oa$ zKjuS-MJ>)r@{WnknL6%4bCR~ivLBFY%0BbU{?VQIXLh73HDF?5+)c$O+c&u(#O;!F z?Li&bpp6ESbaZcn6s9qq?Q1hGU}QUwVR>}3G`}iEw#_~%n&wT3_-11L%|N`AE56Jc zJt9Nky6Q|~grk54&yT_MKw9o?&*+}&;lz<|13y)TOopWEu2Lnic& zo0CJcp0{-s3>HWbHjSwD!)x^3a1?Q+)P(J4cQau=x^O?t{o;5)X~;;`H)lw}2VW)A zb-P+nFLd6${2?Z9qc6`BJ#V9Fl#G zU6zlxYvjk(3rEaRU~l8urzE-k&z2l>%?sWrvopa*4G^~+Bspw`C^yDC&y~_J-8i)k z3-WD;bb+Bf%lUOA44`CM~lP46E=tBxXlOL0|XagxheWivke|=m6&z^fQvzUiggM;NZvKb zaW|mPr`fIB1q3asI2LIJ44Gd3SXK}r?Yoc zFe-%}whMf8EFn>DDlE@-tWs6jX&}40Dxm(AOPlHZh2g7gOrW15e~_Vtu}w zhve6Csn4B_j3$iC3_}<<=qiEYMr52^BUOkV7QY^q&Q4io4nGwYc7uua-NF%%(q6Dc zg`;u^#HT2eJ;@2GghV?F$19BlHN!hSgQ-a_0oA6uFhWaT_tEr74pvC(tC7vgu=|@e z-n#Ck+VT0NgVKWfF#*o_tt>HTVvA?zKpo`7r%s1xPj*S#BK&9ixDAPY03(4l8GnN$ zxLLobH8=~Ku8ht|EQmqYgi?F!J2lK|#~-!!DgPxg$D55E?i7`GdR@?OnAbS}A(|!X z9>8q=R%>VR_v*G5*>pzaQf%i%_5;r-SXHWG{JJ6y6BDRtJ29K3W8~!X=|Qbfo?@$8 zOkwv_T9sU(lpNB$s=}QIxcRw>@iWHdiF*j8OiqM!Bd~*=4ZfxfJ6hjFDAioAS5AvE z*cRP7RaUqhm_uhw(TG(Jt`ctP3%$%iX>=^n6~fyRVDQmmpRm@upg;cva*%~C_}NLs z9TVRQ8;FBFxFWVZv)3gdWgxh;w!|8Z%Bcy|L`rHFb8gAm8c`mUB+Q=lq5W_;pOe6z ze{!Khd{0tGk>ubihq-kElFIi@=BRm|qX|8_e|^&Qiy8#yF+7y)$lrUpFwkYO!c}2J zl38ol8O)ME^Rj(r0(FHLQBF^;RY#y&?HCuU73^1r>Gb%(1J90&wywNAc?pp{AH-wT zb+Jp{Ablaqdg73g)&D%%_w6~Q0cm_n{6K|Sm0^PKrvh2Fxxz_iGE56$Fh_k4Wdp~D=_j!KM z*O|LTLQ*XXKm6#Mqvj14)<-jvkCq+_)nbD_A-AeeQEBC z5tnPdQ8H?1!i(n04ru#_7CpUnuyf#52n@MSFBeu&44U(4Db!09&GMhCE{iE7C;L%D zZrT|+xyt1GZ{!}~ODT4j3fzpNv~q>MJhq>TOO%3oDX9Z4q<+D3=I9Jker#kB=hh zxSoJCh+0@C$ml-9j`=t*`M1vc$9^j6FYNAIS;kx^Z4-Ox;LlKJ`@D;_SU z%g+fdSux|Ky98Q4lVPKpvq}$MG#0O3y8Ic+i0o1vEM^^%Rpz$#Q^H|py1L{hT>5AW zH>&uYr_(C)YM%z{bP=(}wX@Fle$=SVfueqYDH_0eUP(K3rycTa#oX1q#MvjA|MRAv zkrT2}SrJVbG;=$vR8X+rRJ^47`ZJVjR~%AIu&lKEp~ZQz#Mp%rhwvVTD=7gr=I`}~ zKq79zsJ1`Kgf1x#K@@gxS)&7kv{(=(LD#2dS}Z<8fuosF@2sVIqKdt2atEd4D@-ls zy|gzSfbEoBM}?zQA;q{z`O28n3aGN3Vle1lnJGEoUQ+VN6kzDc%Y6GgMh|SGKQH7wH6~s%^i+`AQy*HP`EW($n+kB{|U!cwzjRoZjTtWU} zqwNFNkJ8Pbx=&o-j$rZsCCbM_y_5Xpr8ny&RR}Gw^8H8Oli20?_%n#?zl3F9>ppXQ zXjZ<$Vz=Tw;$E9d?9BZ}+g!%@LkMFRnt$dUfn2I~$7p*vv^KnC(u%JgI|n97MV*=nKmQDUd3L{Kv46~VYK_KR|C!wJ)}xmb!O=P;;Z!88S3X3zuo0YZiS?HUdQq@3s>eH47YWz`$)esWwj2G^s(lS)zgk)A7P`+%Qob zQt7wweDbl{-D)M-xhG7z>195jx%WJ;(3H0u{+flH%(@?syQ*CjqG^pDN~TC2_r7QE zA6BR*?{|rhaw#aiURxx&lxn6T?c_loFY5i#Qck_JZ_UrMrB$K_VG%=H;*^w#PsCJ< zABFS~0?;2g##wCG2?|P?>mh;jHUVao@sA^oM23c&TugpZC&YtBl2&&OAjtl*Eg?OKW?fHdsndF7Z_?H)TiQZTjR2+G^j9Xf&WEy*VYGW5W{h*Z1X9)M*MVZ#2l{$uPA0AF(-z;VKNn^S=+jHB~KDY zL>;S9{lst0{Fano)sJD_ayY<3nNV}XDv$pmT}z2wHMnZ8*?xtDVLO9P5~4-Q=RMT{ zCf9A0@dN1_CWF|UP$Pb&0$m9hMb*$*X(Hz!vq2ogj*+~)UAt#Y6S=T%#Gb)QRxi^; z(cB)sNIOG$RjwF|yAp(3=LV4%fl~cru)OgjwpbFAPV1}uvr;>Kl@(C>gIP&$ZxC3C zUZpd9F=_oVzV*w{arfFo6OFc1(M4xF>WmKua0vH4w1lVP^rQw&R+}oXMBwX^%LKr)KQXh5i$J_sJ&fWxNq?O_#j=%HZcm zjr*50Usez}>zfAQ``C+y7lIas#$^$c=jpLYz6)CSf<1wCd^Q{m zk(h0_)9-TK4Y`~1p88{Bg7rOWV**c*SLelI&1e3)u#G}pXQqi;+jbD6>|}w~ROj+G zy6U8auRF>t4Dp4Hwq}MM{pezPi37hGS4Ds`WEd^@Y92v-BvqwMcxpg1+{Vi)H(8pn z%%bj|zI@e-z>kcX@Dya^Eq}#_yxexBl>%dC%rHgm$9OJnzSet@7~nhI`gP$gq1AM= z*!viy%&xb9abqFb3z^aLjJ@{!qGxwi3`)^C=h^u|{l+?)Ox~?H=CKsWaM&fOR^0_X z_OViWpT4_lnpaw_lwgn3@#XB^Qsec_^_R^=*8*2|EUM5Xtx#eyb&_VQm1ygd$Ak-+${Yuthu-B54S7jd*T{L6urLZl3=C1X|S%f-BdtM%y zI}Fqn_AetU8tLqbfv|9*D|KlQwbX$c(NuVvw?rLTiDyUKF=Wzz_CXIj$3KzFTTs9i zcDr!fNxy^DM7BbeKCrc$0-k8xQNCipB*@I+gHQ*4%X4KALkyWcNL4ee>idIyzMO3# z!jM>ri0%C46O=@)ja%xqu-8F+qWrWRIj;#2JS?%ni~U?2zffJ%ag~Iiu^m8qByD#LaFS z=w~Tk8xf2`BTtNZrAeuosGQsielac`@IVIis+3Yl>O$U-q09Ufma8NMrl{o>F(^`A+(=;y64h5W4`W8ZN6=1*3N6=yn%f;O)QK1Yr4J7k}*qCAtFtW1g4(v-ehsCK$D&mMIb+3clkK zzrdKo-aml<;QJ3ifUV7ofFhZy`Ul_tcWQ2x`mMkJAkxjW{{MS!{)_eiz?RPc(iRXS zznthd+5eTlIA9$A6YzgqTmMD=cKqdcX#R&&*6-Od@&DTF|4-KZ$v0A2&G^jlC#1%& zH(0Ly#6=yw$|2Ii#l3S2X|k{F*+SI;z!lo?y^5EyIZ4G>jtnn`ni_=qhFeOHu%#)gu{mD+oY!>%&Q#(;i&vqh25^FnJsFVz|XDW zl1YFIL#!A5u4)8Jl#;4f*H5-k@)vGtegGT^d`B_|(Ehbkgw6IuajwYPMIJN`Pp8;i zvnvq)4YS<03sZrtsb*ZIHNjfMZ1s%N7I61kwCE;(tu}4qQ~}XPeLG?U=uMkyci4LG z8d0PUGbgV42=RT#O&vCher#4V=6i7WgO6D42(~mbyh-=2P@NhpJ^~qj%8BUI25^Ky zZdPg=-cWsv&ni?NKQ%7|aQf_KdB|b2q2bW&lUCu_rZRy673*-S^zMs3xL58B$S~t$Ue2)RN%UIVKeF7Oi(0^O1o)x-=jTXLfT8R2njmpE3$ThSvr%(& zUOM+AjGHZogB{o-v(K|k7aO=R46L%V6FXOJqU5Xf^n&kOJp%YLU7Q=toD)nSC71|ER` ze>bic#&0+Dx6h@;E^)#;;p3+zw)&{6j%LQQ*-ikP13zA=nAQ00o2>o}l@__9vtYso z2M~p+o&L11tHxW9BXlOcoK;1z%iafAbv_M=p7c(o&!;L#i}h+6y*v$zd0b0v1!B1V z$Psl<7XUn`Oxa;0^iN>=8-1&CFYO#$ET;ts^E8Bgd{;U%XvR~*>w28N$sU<}EwLpN zZeXR}1>AY8(5p4}8Ml7|Ex&E_vPC5ikkmst-|Pn*fPufepWP!7nIjSfTqyPcGj$qe z@qj;2N_7f|+m7JSDKo+I5L!dfGZtnxiQ*l~dz$>76$b?UWfd!6YcMm=vCK$r>)%N+ zERP+`_n%Sn*YY+EM@>y1$WT0?IsKg}M$Nn8%78T2m;TZB#$)bMrB0Yf0Qdircf8N( z05_MgRnX#0lIj$2njk86R3pw9L%RjwwfaLV*@hT5@uIBXNJcD5Df)8$x4E{r_9lUV z9wNUsIJds;ZeEH%ti%3c@S0%5+bNjxyW)$b!E0P+=0#MQ0rj@x=rq%)zWKL0Uyn}# zVk4|icb`K5Jjo`P_}IZ;f|U1Pj_jO0gLp`UI^a8lflUvlrGB*3d_Lm1;psB`^uu-F zx^S4Z)nX!OFe~mvQnglln zE(?a?PQ&6Y!am6!NhoAx7*X~m*#e@3CB46ltL$7I?+%2#b~+E>!JTGvTee$`wupMw z>%f8-JN*>0q@vv?$gBir>F@m3n1UZU=#ek6_4s&);7qI7x%q^heC7xT@WGVtQEGV! z*#6lj`O58k+E2Nx!6zMkF1v=K!pz zQR#-h+57Kie_j6!~ar1tPk>z9(7mb5uKsX88`7E0pOye}dF zIH^Y9xwnBi(m}7-vBr0eUfLK3BLls*-)xBj)Yz=S*Us_QdtPz*C?m}gn*7QLy3<+L(_?hqYNe z_-3fpMQgB@HIIb8fxz>G1;H-hreA&f`j3|lA>#SD;v&}?D!W$4Ia?);!KMMG%LUpF zs7@Yd5iE>6$2g4`>z;Qk+8BX4P(Gv%g>u$9rUTit{=V|hX=m=>Ra8gm-1Tc@er{nP zKTT>l0OsMdTNe}k;y={rP$Xgv=30UY@70Md(KPM=*bEzgE<&|Q9CxTVa^KTcVygm9 z|DMne7r-&UDG0V+(N}i}yL$#PR#Z#5Usv|{JCJ$k<9_R%z*f$$8sc+3Oa)Fj_0d~lVDZ`A**JwYsuEN+)0xssg*fu=Omc~8yka?dy`r*pUlvTRJ zz?xVpBy$0W!WwEnH8wW}!kz8o{=JC3P+^PlzHs0mzhnIQv9E~0UY0A83@KpetFde1 zT36zGpM0;`YWkJZX4>X-mvkgYFW?8xrwt7n1)-4nNxK8UC%Glj!FbVEtT8}dg0Lu1 z55H||`!@Ly$k$;P^)mQVt4$WRf)>ctcAUK$z+Q3jORyuKs*2A5cNj$!pCt_^dGiW3 zpc2d*ODRFev;g~=RrjkVi?mlt0<7lr3kV#r0hGIn2MDrB!LO$WD^3<2fq0_F9)5qP zA2jjs`?J7+>WW&rIY3Z=#S*Uem#bYutryh6k;g5nk-OKe^=%GIwl5jzEI8*%R%h5& zSkhh+#@5A>k6DA=W?@E5Nk^YFbbk^zL55b1nt>Nb=y4{vr@Bk-kGB7yK zO;RQIfFai2M}VDn*JUKjuH5z2Nk|3QslWOvwpd2i^kzh1tnA~IM(*-_!G0$>AS1pj z4?KtlLcaR&pO7np4SCOvE7C>8E+(&ETV9{3+yVl({sla&yXA~}zFxR2N9fOf(mP+n zFG@jkFV%r6H!HrBh}B4VaSi)iI?#**5L2hdvcNc--~|I*LhXzpr`tRHaNNrq_s35u z&i9DH+zX6Hdf09Ox~JNYPc+8%|L{Nbu-q$rCU?eeSbu&uo!mA0Z4b8-a z=n)do1m{nQ|1@<*^exH0bG|Bg0ExCgUup()1oeM@ZR$*P9HJ)3kI^IlaLHUav^3U? zTs`BEA?lEE7X%WXEy+LDJ1YJe>QPb0zC+|Os-?TgkArmPwOEu0=hH^+TtiEp%8biX zudV@QX;yr8rUA6}SrW#cpHj%$(qC+5xfXP$Zt{vaV1#T}*Uyz>`=cS_r&hLsKQ#s$ zS^hbHj0bQru-!^DA-_nlGx1n&a0vPQ5y+=N(|D>tsPmY=P(NT^vl=h`rbHeynEs2i zS7j8Vlw^_#*x*Fu(~-C}z+<`)3AIAt69AuwO%c>)aUW_tphCl-y{zTFaaZI|0`Sp` zsP62iqp^&bh=@I)KEq|hG{RU3p9^rx{;Z7lnWBwVdc3kRT+JxYRRr8SIm#i;HNiwW z(c=`aOg=PR4YhD6bIr8U4vnSEf6Q6i12WPR2`aT~x>}Na2U*C;pD9JT;b0L@>U1T;N;)bK~M9Alo0RRu%OdfWEZzm{~KG&y58ihYiP>S+TZ z3yCAgXCOvL+tiGwAI|!pBfCG`$Zx*^bm8YtRFoO=C4S#4oYcsjoS9d0a20wSG+OLx zjHz6<3!nP#5sn%Sj++-H3aXfHSu~OiYtI68%ai7VG0iCUl{{z$Sf(Pmt~Vz!Y|CID z$SiWT1+%hb5l(3qyQ_Noq_)vf;k!rJ3sTV48$#!#4g$HS$aWSZ$al&FHA$>%))M+H(yyyDlg5UzI-=!Qj zK9OJMa5eYDmYlc+DFvLR&A!}gA(8xKbQ$1Ru4ZX@`XY-&$o5?SlKv<3v-x|e3Ndw` z?%)A68}ov4dS{4Xe+{ND9O}Kgx6L^-McVrssD8nvGAu8AaJ(eN)Q%qgiWK=r z@*!Y*2stROtVD5Q-XJ^)NR;82HZEI=(fbU5YC(%b;qEB&buajFWKJ%8+o#H zP4GZD?-ox%3QO@;weTu%!ClScd4IauchDn_P!H3wl3Q!?);#LKWm_fPyy*@&IcV8_ zbOShRFAz2gb)G#y6*f5p{3TWJ1?eDG4+&sP^%$f|?AYg-_+U>U5sRz=lZ37v_5UL8 zy`!3Z-nYRZpooe}5s)Te0|E$$0#Yo1LJ%YndXW~SH|bRck*<^=B1FXi354ET5Gm3X z>0K}&p+vwCdUo*hEx&#D?0a_q+jI8t7f+sma%I6?57KzS^u{`JB5PHisUWh5y-59! z?qJVNW2&BYHNnWC`Q9Ojd-;s!ld74AF0=T|nu&qKjqe}r(!n|sWso4?Ezn)~-rRM{ zDd4Kz^ZQC?tF)WojmkosWkjJZ>6pr6L^?=S{K-~ySI_T3(#ck0^dW)hC&J%0=?90` z?5yj}icT{?dTyodS-EhH+n=rC1Et@qN@BXqxY3seB}O<2ob_BXMnHzs0wL5irH+)S zMl5K3Cuq{RRx7So0n%tABBPkUo!F=5>|Pp<3A2g%s#kr=rJH(pz&@!Ge-muek6Z!c z9CdX+>`s*W#Hi%C&NbNraJPxY1y)pRvE4e)R^(nklmiN;1CM+H?`C{1lh^8cl??|w;>2+R7817jgB5$pOuOG zdL6*QYUA?sC=j#l%!!&u{rIh2_EJ4Ho3HklAP?J77WcREPg|;=z4cDbh*|CkMQtY! z#Tr8Kzk7}s+_n->t<)M>!KqHeLHV40=W_3gBbLZRkQNj%v=fSI8g4vpB{I#Ma;)z| zVaeQ$Jko~{D-V$F_~39x;F@9K$vx7u>}(!)4ba`Tk_*We*~y&uj)aZvj6XmkjMiHo zZjC<_Ir~r?)WP~!`=mb4k;tX|$jpF^@#xDC$aUAwwJVfV?>DZEn7&Q=o)#3h&`haO z(DQZUMWXu!SgL9-i6J^)?SB5klG0 zgz!P8qmv!RgazHe?Fb;}u1gxCDK#77F5gj7m)Fo7$dNdhM;8wUuj<-2;iz7n(y4^4 zB}RxlqvouO!Z|xI_Ud5lQf*y+*IUHj1<^@Q+hX2$Z3r`@PN#vX7C*}YTfC%<5>ENAXk!wJZAbB8CiR4+oahCWDr_GFcM z;h{fzdhK_sL|u8}ZVDfPM4nbkeckPGv3d6(o8@5|8uP?U2+H`ITdW$m#aEtoGy1rR zL@~(yv!RXUf#pAY$7Iml=8Q)nJ82nennv7m2d4>KPIgk1s%eiL4agusx~tP_uM7!O z!q-ZJD6>F|feY9ZSB)+Sh3{X*ET@@A=^J;ahSMzK9!k55fNh%A4xD7^C+zF~Mw}!B zBGzq@W&|e{|B2S6Hv$H8{DWcX{rd7h7}qofi%ovWcBoD&((P&tP(M(1E+vXZ=#;GN zY(GdB#gg}8c_BC3mQtd~Q%ZsSm9o~5MKEoZ63EKeM~Q7IiD&&GS3+;48-EOgU4$|g z-DmhYlz&QC|0LwM=J~Jo1zKf;QjPmp_aVwV#}x*;0%cw5PSZ~hs{R&{Fi8J3u-SC1 z$`5pRCkF9Q$d>RY7#-6O7LI4)%0|T4f^RXpbDa%^psDRk*|U<*g!KYkR+Xr zO`b_Sv>98(-Z&d1k&#C1x7UMA3qi#zsibG!W7%IB-Ab`QsnD^d7tF8-v_94C(3nT$ZM@qLR3OSk4l3a3}rf{g!!abReA^=joq& z+aQ=b0()DxOg{6sXwSUY;^nKDsY8&rC%#wGn}M(|>{%#Fh||?Oce|#oK0^Gu2ne#0 z|3>$Z*cZOOmRI17cwiaiB>%3_M@%y%x5Zb<>PI1dSEY!=-_ZerB4MQ;R*$&*UAQ{1 zqs1%ZzbSM!3~zmry!qkLvUFiq=KXQBt-iPF zWc7bl!LL2F(#ackSKca?I)%YamD;gd>tFcL+*erjY550mFeV zErPP9Yg;7SQRlEWuh==uWH8YiRN`b`(CUc1#LZuOu0gN(PSoD?9c3vkayQ6B12)lhuj z^^42Cp)iE`XZ7~4rYS1x0a}nI#49%*jj%DcP~nQ6AZvr$VZF);0yx7Np~GP^dtHi~ zJ$fg3A%5L!u|iZDxpQ0PjpZQ7=zK|ug3MpYKBsJJB)Za@dx%#%$YJ7*9f{Z*@~}b$ zVp7rdd<`VvX&XtQ`zvIb00??P9Dy@DTPH_0m$PpE6G& z^A3v~SMh1uu@@VED0NAL#6y-ixm*3@^u}ZRi)Iawl|ZBl-KN%`GKwaoz%bm{GQ?+J zr=pIK&U*;r`UQn~Gf5lVn<`)8Z=}={(nGkWAAj(#q~LteEp)EE4V-tHQ`S(k^qb8n z?z_;q6Yl#K4Jw54-oQEyA&@TCK2Ck@Bd7fMo-zH~oIBzz_9p&ep95a&l8E!af+y2u z$q?7x_o(4`<(IvijrcKN5#=9xVe72qPN^(X13D$bif;P((#{V-*3oRQBXT-{l7DM> z7Co}PQ6F7zR`&2Jo$KA8Ew9`n{RQIdTW6~5cAzbs>Pf$&N`^h0df6# zr}>BS&DTAA)8k9E>}bgc5EkKvSovm-c#6GS7W1^%o-(J}YZ2ZmZ&MXCVFzSqvX1=T z{4nJC;#glkSPxwP_0{lVO$Xm}*W^5TrJ-o3Y)vwB>hhr_$%Y5=e`3$6b4>g2t~vVO z5(zUze*^h>A1m%gVsvj=m?LT4Lufm-oaahUm?Z~ zzYhOS3+Z&eydMGyvR-6FB6;cLBqR3=z8`-IaX%(-jwP)s#FJLcrWRJJXdDzctM$Z< zM=9*Hg6+c&)?rAcVAK60)s4%2sp!z-UXf+h22oRM8zV?HpoXf!I4O zrX(Vn8RYuxf73H&Lqkuv3!~>juH()F=4rV>i-&Vf(qS_D=-}ZTwE_ z2a7w8naELhi;A9YdlSE~^xgF0p%Nj&rkKEfl`g|0(=p+MG+6A}#;s_w*vVpC{IPxz z7PUlWh_TVRJ`=k@*P|2UU*Bu*@yQ@;?C<8*Q{YHSk!M(e$Kc+mpFX#LjY|#k`_?Vo z+36;{d2}yp5G1GA?O_MgAu|GXWraZ4o~cMB*le(c$y@GBf0pbv&;Pwt{q0Z{@2bCF zr~LxlV1O=6j0k+*m~@~uSEY;zB&Ss8At~c;Tn8C)T3&lVT(64}fUwXkc;1VGc8&QZ zge;_$0>%=E^iiX^dCL@dM)BaqjwB{>^t_x)$X8ID|;N%(k;hz z$;u!cq4<8-{t0mp9+YVN_~Wqq7LxGjbTp%1UGFcaoXDXn`YKoQw3BBJ%U`xdYnO**6hjV!%KorutHUqo(Qo%$(@;ipwf=>8mVO zl_f7Tx(BWDlY|(<7CAYY3m0lPUD+W8u?NHYy$>GrCKQt3=JK%P1)W*p$aH?giN<|( z*wwTMp6LL^{05ogUB~{vTW9JZ82+*p2{~~+Y_X(_9X%(|=x@aK+A~GCk+Lsd#4Fag z5NlxbRrpAi#A<^wb2}<}j7uC+B_v~6^Qel>EN3#63}lm&EHzYUO;u1oHoYC&P&QOa zCs(%u50X?#;^7Nh{By9Ig#2?jjAyk6m>%dw%)tf~9AX%7Xt>(C6(J3opZ{?zJ)$wW zcLg%!P!?W6ya~~!PV!iOWcRF z9%&q82ulembfb3f48>eSRh>!+z7fB|I8! zfx_{P{r_yeGEE0G!v(glK@zeWhUNChUO>iCMj?nPMo58{)3wuJJjaq!XS^a58i|j( z*?Br6xIkA{%TFrrCW6VB%}t&8E=gB4L13h4J=^C88>*d#7hC&Zq0A3eDak7?y3sY- zwLigDk-p_Z%o23$p9TXB&l{%W|FOQ2C;6rR2>p0L)k8t$gmi~fkb*`hy}QhsbmJRS z;f9utMo&*pOAkB|Arm1(f49FD#vqEB2fKptR4ZXR_pzCEzdvR|s!mryy0bo%x>U_@ z74jase3}NHNa!8aYDp!j(@!fjb4q_BoK8OErzn+vk0Cpj0U~%YtLW-Ah*{A+sT&^*ZRy8Z38HPKJ&r;@m?>80i!^DVZ2-AcCp03?vw6iV)pSHHxhgqekvPa*uvJ zWERP!Rez8!Q9slZIhP0T;1rm&Pl0#rlvABB$!ph{LcgTIH7EK7Ja#ld9b_>~GS#X6 zJow3C>7QD`lYhUq)n^9ID^P>MlMsCB$7J|qOntBD|7dyt<-yySYH-YiyaQy}+a3+j zklZIs8E9g&k|n29u(Y`>vM8qCIF-7nLkgt@(m?B2w$%L-N4D@ zv>w{7FF1X0e-IOifw)E>(~H1=M&M5b9QzFiXKFN!n5}>~@Fwu5DRKeIcn*q5JrBhs zv!Ym0;E5IZJNW1JL6PKx(%=DD8UFM1;FY7`tnUB&R}MbJOA%F{Ra&H zK?dZXYX1$3fAt!G3G07{{(pz6|0u)1 zl=L6<_!l1jZ2P)kHY>=@qZ^2SfKw4sQx{F|3<}sY{!2m>o1z7-!#F&enlK7 zffiS&+%K?OOE&j*7s|BsFIcUM(D;i~1+8Ezfo5r`NOJqhw5<2TI zXmZwe2(%sa;K`8R^b4oK9_$w@?9|>$r9}{BK6JZKwUb2BgK?koA(OdHtwS}9s)%8v4BzgbS)Sixjt0B($&R%I%!qj#i;r9%c*5hlj zJ55$1RG)*z0#UePxY7eGTY4MTm(hV-ci#(KfKvRdu(atss^mTqy=iE%yBhj4A#^2} zaw%j99lA>1bqf8_;A$-(wRw^@_h+xg1sz0O@uC`%(3{^v*M5YA z?xho?>d<}`9s=M7j}0#g9lVu8i{m8knv&}`dxmJUh%rBLie<>ArOJUU+gu{Ek6+B* zIhd$rOWF*2Z*_=NNLwb9Y%8zqcNkjMCJ?r-qnVBKbGZtoeQ1BsA?rWtrS~iOzZdR} z682^!_vQ*+M^%O%&?abG==#kq>i+m=qupZ3-QPK(dL42t!@E5+N_zaI1B~>eqNl&+ z+{X*-=o#|8eKBvC>7?jpmDrZ2vb#~pTp(hF6=McpSU5u1SlDserAmD^O5iZs=G^Zh zJno;*MN#WlA2h*JIbQ^-az*Tr5`HQH(!aEkY{E0Apx+2R1k0KplDZ9FH5*zgPEitL)wl_1pG7`{Ol}EZ#qsX{1Y8eO%({^~k#3l8O2&I^o#ChJ;l2A(1;5SN&G& zQkyIzSu|R_%w>8`UuI(tsJm&JT-3k(9JA&meMrQKoW)Y8s^+Kp#oBz>xqB*!i#)88 z!=l~dlNm78>b1)jx)!%k95tQ%wSQKBDNjzhKpct<{cQXkL%QBDGLqb|d6%}Y6FQ$0 z^5?F_u^c8CpYr3uJipz-{e{qt7Pq~06)#GuUpey1&V=P=b>wW}*lv*gH4mO2z<~{% zgx#0z*dJEeJhim7u0rLs*d>y8TVoPvk%Zt^R#>L?X0I3XG64l1>3R$~EF6Gb0B5*r z6uMy#=Fr7PJzHV!N^NE*GHQ1)z9-osY$pAHJbfU(1~g z^fnf);P?PX*;cA7>Q5!?5`XNhauPOMBFBOjO{c(NE^-L9MPUDjz@&ceg-o*B-k@8s z#2!Wvu_L87A5qBOq<_Z~;!IJF!ZF(IT&lR=yQPJ=k|etDs! ztD#bszFQ!vO<^EbleqO(G?x8S#UK|d_}W?Ypj_T;*+U^F<_|;l46U^l#tYvdZMl&TXJAmUo zR(8h;otDvIqwMnn+N77Y8YNXXz+J^(I%5QV0y5fiSw;Qu2e|Qzw-t4#w$jiIq-j+} z!kXDhK7%($VZkfRjgtGHPkzGcJ%1KY*}cm_m%AFl8Sa~yf%vBaTfV1fh zfBdVcUowcYH_$6}TW^RJ)j<(fSxd+Wx|ZkHpqq2tN$s9vca^Hq%24QRC1V8td!hU@ zZFlPoA}|udP!1V7;=6vSR6?iVN549zf#jlZpkHq+`w2l3rxkAyowbWIbaPnS)Rzho z^Vi%O99pq=S)WwMDQo+iF~iK&jYN;Q3B}4VlYWM2TkK0rT|~xrluDE%OTeYm@`_9R zfn|@Ue^v8-5PC^jn8LD!V@zfR=yC^H!&gf;D-|VjGW{0^#o|N3NknEC@rrJ^MSzTC z3b&K;_1VdXhuNH&p&J*P`?@{r<|3?MniOE}xDkMP!GN>j{2|7_!eLv*%ZCKa^rY?>t?&8~U?+ zHR_?Zle6_WuW1f`SYYUC=XiM}#(FM;^-1jVES@q9$FOjsq9L#@$Yb0)eU>}rNIr$OBK`2-hV=}xB7@F{hV6kume2aDZaGXx zT4D{Hkf&~_%8A9l8eGS>T>|$AJP^n?HOY-VvLZ5}Eg`2t?C2N~%->T;*e{JCu+hEU z_FY@ErsTsLA&8s&84eI0U&}B^sk{qIU z4cLl3pojhrozR=7L)uH_Dagyoa3Gsvsp z+9Egz>aUL3q_w@Z!gZZ}91U&+@Gk+|AYiqp8(KHnk zYsn)d?eYv#sc0LfsDCRG!!LX}w7~|~I1PP&iB-bsXfXhbb&1X!8!O=RM#1TR;{kOx=`#d- zdzKiN7gCo2{fE^0Ta(g^fic4kf2XOGv7ZmELR{rfr6#=EB$wc`f-~DxTuoP;9z&18 z_#~hy_DB<+IuR=jldRe=oNS&R3rO@@U9Ko(AvN3ld1_I7q7=rW?k5&c{*99YWJ zVzYr1hzVw`jB67eMn+|ae`WlUmUVlB-=rOF1$Quq>Nclo1|jya-`V_0eqEEwVmsyK1XKlJe5QCwPz{@$ zR;RK?iq?wF1QNE4;&;&U7iacoHi*>O&`zl;WHx>{d)vP?xWXcx=-*G~lgh-;4+c&Q zkm>A9Q=1|vn{C&UXT=?n`QW(Kc2U2`Gx(~}&N#J!5E^BLty47LFY)d6Yqlw<4(^&R z1GoNnL|h5T>c-u1WU6-_y}8{WUUtkccaM8E;05tCbt1cbHYAsN!0W>FDPz`YYw$B1 zL%O2Nz{?L@(&y3Z2x7IMp@H;u&wZhIaVuX&D zUHkgS$O>CkC|j!34Wg`vnwj1X-Bqp2);vQD`6OU=sHR=nh{#Nn(x>8#HT+K*C26#! zYy6%*1$TI1#R(fD*}TSxrl#fl@5q}r)<=X9L^hX?CRE7pW&m6I|Ne{&JW;tpSRyC zx1iisEyl}Knlc}cLp_QObrQF$a)RzuvVReMQv7YNdAKq@DZm*f=dekgCN}VQx5}0# z*duS7j>gyNp!iG6Ls6}v?2kKVQI_;DIxX^0&M}x;*vDNOrt!A^(KOYs@AoGxj6`pr zz}x-wq##*w0>Udf99*Lq;zGcusLhFQ70GFDNOp%!pM0pg?{u_kaF0@92kwn>ypEo~ zA*1*G(YQUzAzv&(BXa^{q@%fepVt)~9&S?s;|qEqUl` z-=b8-d&+h+y0@nc;dU2WvTMbky6tV@<`&|L-%Cc%7nmSD{+RSo`Ns4mTl{jT(~I

h8IgQM71^47}Dpx6^SXZ*!_u6%Hv#K&frKD%$--W(K~HS{1Pqotw(l@z;U zrl8%a9Pjd_w5_G#1-i}2`r4*`MV7vQ#+-);nUk_F(P^Q@Wf@Jmy}1>5q+U>Sx+qLB(h4?S?idJ^P; zQ*}dd6Wo6gS6BCNH<0D9TJFU|lJbuRm^YG*qt{CO@Ww%&k1mJgbg}m{ua!N$0XW{4 z^Wicn_BV0%7SXF;#zvSY`>DFfL|j|wM)llC4QZZ_M(C<4V_%Cw6NAk*bjEe|D<@f# zr3Sr!=PnTjgZrOg{od}r?VPQQ&k2H}jPquS*SuUKCP^NhJ9t5#UWoh~Vq``@6K^6<%qPQgx!c>27{Vtn`>}W=l@_af`GF97l~KqwkD8 z#f7Cl)?%_XU=G8TplE>KGmVVu(1KGo%WuQ*^Ck%zlbjv|e~mA>gG&$mN16g!i>jt} z-Ri$0V!iA6Rs)OqU{qcHJj&*lU&pO{zjzNzI}{dUcV)e*yve@_J`b4h=pURm;M%j` zOMLPBGoTje<sJXz&`NIAn17BDq$vhcY~WFxlK#IMiP>X}g(*!FFk;RAM-5 zqJz`y=c1P%P23$JuM#LrW?QT4&i42rbN~8&nOAY~uY&3%7&n0d!WQ)pa%0p5(YtOQ zq4M_<^El^TEH5$!d**)h6}ZqZm2}8eiMs6@_~aFyS}zMYK5m`tey1*%D|-y#w!21_ zR4NRBu6bO=4Wj|=oA{?xxx)c;m`(q14`U|?C%prSiywd+YYw9T(WQ;SAJ|#z@$;uq)_ZqXl^CPy(!nv`o#O4aDp2PV0sgf@9GbT2aG4Wr=eJZEN>eA0?CBq zNz(2PjG&jGvFj>%J|&Oy12%PM%*E`s$K%-MYZA*j6HTls6mr{Y8ugagxoZK->Q$F3 z#3J8%b{}4;Pz0Z5ttjL3zAQ;dg(^|2XlXX*TR&C`tgmL$J2t_Wi~5ysVRvgkkDP^b z2|r~6%{EjlHq|4pC zH>Lsg#98@FYL`^su@^*-PVy~)OQ0jA=hYr$T@UPjFM8yGiHztn;7lup5#2R27erL1 zh8q;W>o{qxIV}O=m2B_a_?pp}`{<`ylWv_XoqRB|UD%tHj^RJ&o#v$dQ zH6#C{HB83_=sp;H6^Wemydoy>4F44|p3S0I&VNaT9B`&I%|@>G>|j7_9UpA=jLo$- zj9Nyd-(Uc29lM`C{x$vW4W_LkJW2D4Z8zKloM{=LOXbyW|M8BOKPF6z<+3=Vet=bS zfO_FzzG<=Knu~fu61r0m$1?i|8PxIQeNg&f7i{$gMe0;ANGgf%+m7B*oYVwA=fWNZj=Z~&f<0g0l(Av*O zvd(!tB!UZXC=JzH`o@n)?svl_!8+diWj5#8X!L+pRNAyo_*K$aN?L@;r}&XDUAN~S zd7EBWqZN1b#>s%I&ASmVyCL6|WWN-BMU&-oS66mFE?}xW(>~U)9*-YeI)k(w%~@Oh z{zgfhTsXM@63r2j%gd2hKdbUv-Zq|FGB`pnjIJT=)F?QGM?Y;m6B%Ji4;zkzBM zcjf6_{z?b-2_yguQg2^iDTqmB`fp*GOjWCdBY6XEz2_}>Q3PW!{vfpb>W&edA6#~ zV0No*c|ACJ=ziLXyh(z`*xE3lB)@5-sP2zXo|RIW$9RYdb_HC^Q`i-OQjGnwG98f1 zeSs9RdXdo6bEm&P>W*ipSwxCjrwhG%b%&Q|Pgy$1scAhBRiR1mXftS=l!~>)ssSdl z@+K4chly>16@o}<(NZhov}R_8Xo$)Y4^nad09Ceh%0xzMdE+(0@uC^3QtGKzF`>gF ziE1oXltlICn~Yj@QX2YJ%$8v6W*BQat}}@;cr7lVVkeNg$JQ%Jp0#+0Q#R=6GV>T4 z^nf4Ivl}P5T$U+P8z{^ho{~R>u#hCmCQ*AIjgyRfNh4t4)FTNUFpV$h`pNeGeQ=ZB z=|3-~5~5$i7_vz3V7tCf$ajE}$@|5HBeA5HU6+HB?H5N_QKR>vR202TmK`lX<{Y23 z0oH&ir_$M)KY{mb^lY}bfP8KRJIYb~eHHuj&RKcp69c6R#IF{7uVdyXYZJEp3dlBd zT>MH2y1C*iIf}0J7x*GD@oXpswU;8jSG|7iNZxuqC~bXu=|$QLYo0;H%6}w?gaHX2 zX*4Ok>tnvt9ggu~E8Y3oIR0paFr00}$0Kzn%Q%lT{UAg_{%U_;e`Br-}Cd2CoP6 z?%9L-q2Lg{A*z(u@!kdtR6J(lrxq|<;|FAE@A}^q>GL^% zu}a|m^CbaI&XESoH))NhqI+%p&p zgY7(A*s5%OUyfeO*FK1TzV|`zIf)6t>eFI8H^mc-S^lU(E>9WmYO~yovq&Ib^LL&d zE@&&~n8eZFlM2BH*IlI_=^BoWPfD|?+TZ&Yc%Q{WJkC^ce$q1rP{T!HJYQ#V7Rc)x z;ljC1VClqZ12LwQXV>gTs_^fzHekx0jTDoX6I@ilNxUPmYwPU^N^BJl6UYU*&d&Mr zr`p8?l1(AS%r{9!#0AUCyD09G`b-7(ByW=@znNk`(jizjm~W1^TR}Cs`y(-fQbwpq zUcMv6d$7WRptt>ngfwd0HZ?HeRh!EKmq_J;_^b$2mQ&gU4Ey!h^0;sb(m4%D{nD#r zU6-MVBjEwPR>(6Sd6ABc;Tyn8+QetxRno?v)5uK8mAx_0=NW@1u2(Eo?#g9xf2hd) zd{f?b=vaFSW<$NyMNLk`&57kO@sx>^`bVZq<)~_&h|0CSvWn{~rOK@{TXw6%6sfNY zsKuQxZ5m1y^)`NUvMKh%XZeJQKE;3-?;Jq8ixG|?+obd3^4ZRBAlb^{N*-rti zKaQfUFq*IB!|PWaBa(gCWWXo6Ww$FQgT*6~S-kUm8JtN{F5>s)*gQMhdLIP_?1;O; zl$F(QRexJ@rOXn8rJ2oIigi&v`P6m5>B*Dr#F*zf>~Zai2(;yX1ZM5HoRhsRAe4=M z+12Gke5uYS$31bEW%18b-mH;mAzRk>iLW&-C8$Y4uLe%p9nKsKCi8$mYVqf^#%_Ms z#`oF;g^&Gj(wsz~2$T@CaxubiP0KLL#;UB|6vyI7LD4xgzO+Wif&GEsMtl#VZM;PO z$gn-{EPSF6#y&P~8ghj&xjsB-Bg2fPwAuly7`O1qd2Ms$Vk#N9Zxs+;bIQlM=Y!FH zo%s0@-q=mOvsMjw!^c;7&fG8n31Xm`;{MQ$6R?4?S9df|_eF&!vIZM{X=cl&q}|Fbj3_%k1=s4y2nqAVZ0JlgxXn19TX%Py|=sPiz*A^w+KBF*aBQX4zpoqvDLN9eARRcIGoy3Q(@t~sJ+oV3jvrcshB6Cc5 zP$q099aIE40Wf8fMxs%fcj|T4UKgajcf1CbeWT{|IH>>q2f#-fKrJ;jdhW4YO!zYl zE6mw`-P2~H^TdB8BpBVoVV-D`S5v3d`pc2F$LfM*FJx zq?-@2cnYV=${1`J+>wVgvtH`4FvVpqK01TvtszB3Baud=h*%`D%EW65MpI@!IR}}( z3@v9r1uQ}WK)b7BgeECW{iqVyEJK${oQ1+T6i#R7KCtii))*3s_|#RfGTg2sdtRwZ{7l{@bc!efcn%_sIs+Sq*63uf{uycdg#v z0V!3`spL{3l}6iGxp=q2cBC`pc$nd#sC}_-AMPZbdA#uT!>3bRO!N z-I)vq*`4G1uNL$#cI}u()akWj^Olm;8omQ=_F2lk$=V6-f@LA!ly^upvBouzatWrC zin)FBm&yA9>TWW$JW=j;&+vqoep`D}3skCAvvu>RF)qmp`JuxRpZNt&ZlI~6{NbOR zQO1Z;ohf_S4b39#00#Fic?ieWP~&*N*516Fx2q2^F{PL~VVoJG+mV14XD|3qJJZnUY_^gyuTQ@u-7h+7VZpg)6UfrACCL+3&ZNc)reGt6l?o=b*>qxk#dHb1jV^ie#9 z(i1Dy+M*7xFb&{a_oMS_Y8h!ay@l*lBhq!fFEyu_(AgVUkuHt>9;jAJ$uzgag9cBT z&seNxX2e#ozEBp2P3Q~bmX}>|?n{rL+%AA} ze1Ms;kw-La5nb)e5Vx_si7$$mZhwt}4lK{JNjV!jI9y9rRL`eqOLW;w_s(Z%-oI4R zuemuIT$p=%wDxued5BPUQ$a;xNPA|^5gnlkok88@ZReZerL`jrB5trgEDo#gAY z{L}SkI@+j8o~u??iKB(GBQ%RF$DCd5HM9eySUvUphKfHz{UCR!5m=1m>;3 zYI`vsJ!H5r=M{)Te}PxpD_U9-j_lb|q)OOBV8)_`S^g&7^Wf$7)5cou)I!!aoTK)6 z+(6e{Ql9n+d17|rpaV~XpjI$0w}w@j+hzTwx;?kaMSec|B{+X{*mew4MNvhg0qAK3;Di>fA zOJh16aDO-ZdAmVr*9({N=X1;i9jN(vq9(>Q9bVXhH$EIFq!U^#C#ie{aeT4ow?=xy zNLtU7it-5Vo8;JjkTI#ue2PsXvv8wE&ik2ms$&~4O4w>u^yR@c{K}?Z##^(f`JzJiZS4S zX7b!x&bQq z~G;$S!?}euLYkgm6{1-c)kwE|qlM7F)Hk(16&QPjcBbSf=?5CCE%< zq^G=l46r;%ih6N1i6deE`id$_!8je5jC|aHOL-aSSjfmmMoIrYed~bWcSJ+}OZ&ST zK$G2~Nk*kR4g0x$Y)>`xrK5M^ZQn0OK{?Y8Zjq2DtY&2C#$GkFh2~y(M94C1s^~*#3{`{WL| z@oVhbJ=gCWn#&NI<=F;dsT^Q!MQvLGXGULs2A@-0{_VL_Zd8iKLA`~()n6PyHE5t} z@X!<)r_WUVE9YUaFa{<1t|KX!T*1$(E$^VZdtNCRPrHw9$r#7Qnr~-{(`j5PE1Ww# zy)uard6!D`&4-2rIG)^wcTn3Au{FgdAF1tUoGJ&B3{r9%)KIve&o^O1|E(A&9|< z)dS3G9t1s)1z=IbU$b#LU8uG7ASWT7O}@fd#5ii-FrL>(1t_HY?$1)?Vk!}6;5jP? zdbPy@fj8-d3o69o;PQg};ad8p_Isknj)>rSy%ZN@3aVyj{)`hr6F~Gl7`bXu{S@C1 zZ@v~B8p^T7p0d`q%^$Fs%3rweYL_vPW|k3^^Dh-rO2H_9YxTGUo0xnCrEr7U9XT3x zzEb+wdwatzWSyj8Up0qHW(QpM#uGax^Goorz)}@ki1}aoSjZN&Xl_i>+o@n9WJ>4t zJQkw$`%k{GDb!f3PP&#jVdgnjZB|Xb5g!bhJL?-H9o*I4l^~0aw0MTkbCeo9J-# zuiN$)gzaqsd!hKJ%=4=@szbV^;QlC7kS(}G6oEqF4r=FU>eOWEJfxT3vXth-k>T@H z6ZO_-tv8=zJc(o@BadfWb!`jgEBmo?(U=q^cQw(jA?N82((x&S!oKHZG#Z-HlDlex zl5up(Pl0mub?OV{r&N1LODUTgH>QXD)nW*2?e~w8X(aW>snvN}Yti!^;PPF?K<|b=fD&p z?67{zSMw^6tNGvuoU*s44t42zN{VY+$X!(Q4nF+tC{oJyV^wyK;M@)vJE~}=%*MF} zi#$jCTzYR)fe^Pbi9k!LuiuJ&T++W7DGF<`pP7p-EVu*!K`gLr0J5A*!ON;p`=>e!#D`n5_pnsl87b zmQh8$HpIUuV^aedcU)|&*{5t-@4mK~hy>bP{k9^*mcU)lv(+ee~J5_pWDC& z;Y#}BpyjNQGv(iHi&uSIzQApY5w9nj0$@g&~t_1^iqh$KWro`%rQEf+l z88jFlwEg)~I;IA~eYl8R+;z;5~pt1W{w}0haC4TevE_W}hll(?ye4y}R|N8oU zV8rkL{7$CVVf;A3ZGj+-w);|!zXZrrcq#MYjsBUS+Xnul$}4Oo`d#zpQM3AgzNC~} zgqB_24S(k$y>SDWO+$mB+=3~`$=VIy+WB5Q#Bd&Y$rs@G>+ttJ1 zL3qK@55`=O&@I|jFU+!62KD}q@-mEyhEf87C9?jOGx|HD>|Slkb)4$s@6RPW)u)D^ zOU77A4Go-Cz3?Fizw0v(HC!}N`m7zB*M~@{(Pp)2ma%;`XYi$S-KhMphAzw#n0%cFFSt|Cr53?P4lkY#tW zrb3X1IBy!|cwiW#hc%^55ze?i1itmLo|c1@&pz$_tuw~B%uhK#WkUZd>%?N7 zR|Yi^^r0PIuFxN+pV$%$ebUc-&-_AGT0fka0Ne%YDRSmZfk}Tpsh7>zae&$VTF!S- z#47dsUdN(YTkq{*T!+DV3pBp%dOO@-0zsuh{n-o8r*w1o*xOS5+XGX`cTn3b z#yBTFAJb0`7vPUmJPp=*v=iIO-M7ER>9h8x=7~gftrr|Dc(kk9;X3v6xHsR~j716j zB_m4~2;0T;?`!z4uk|=4cp^&8TZ#p{ycbvRm*{8C-*7p$E`uMu2n(fnmEZWhRaek+ zZ=x+gproHH2DAHgp5e8il(*+sO;pEItryViF2vg*{)Q;rdb{-lSE;JJ(d4$HNA0im znx_4Vf~H{J8FOTR--s-ioy0@SM@tcmc_aMDVrZt50S0$5Qzgd7`~F%09_Ng(avLyM z>qTMK%(PXTljhCw-7jBTi?7R1X8bU({gP5=sFGzX@)E-m@!}C?W3;Sv2T{_meQnFO zro#lydez591^8%6hT-=Li5PuE3ATMbPzYw{V2>Ec%Pe%wS4=hahaL-E?X8+hDt&A+ zl53jxy>}eT?lYR`dOa1fwvRV)Z#7;KO3H|dcQ=38xw1{&1|vcSD2FiBGy#L zh<7Q_@;n0>Hoe>7FQT;sXFyPfur+{dTcm%-8TyO+l=K%c_u<|Q=mGDp<~<(NJHX#u zgA`c;L5{s$9sjyQ#=5ISC0M-rrP{qok5*SoO}P7*z15)i1p=6d4co@a7=}U z*uqX6Vy>I2SkiCt(Ww2ZE?8gR?0|QJ5D>~lEY3QscRgWOs}>b(e17{QltX$%)3al? zfFZkPrLsgnI251nT?#&bitGCxSC%A}hBuj6w#_j4K{B<)Dx?n7kViE#3+|hZAuiUw z!uT}V?sffKzlYl16NTC7&gz%sCL&UFJrOx6xp2FWqFLh|aKV!ZU0mKAdtaZ*{_(BI z>dY4+0LUqZ+7<$fFNd5jbn9y@tGee28JH(Qty)65aXc+UhSt3Rz!YbEbBRah?2`U_ z@q|*7^g4+x-XVD({H`{!-kegbQ_g=LR(}aL5PetB|E>OO=3BWP*JO-w2A<5iHiq37 z=Tb1)n_g$Car*3qXnlOgK&ZmjtO2-9ni~zDQ0SL^t>J*ULTbjDJAF z_nzM=Ip@3~_bF&I)VDJ76Tn+l71qu_?tWvg;dsGtHz5?(m87XW^9|uh`w^CIl3wJH z9{s^91)pkuH4Ub7_2?Mf{Abd@j+s_&|BK-p^;qe1{bqn&_Io+b^&HulT-#}oqT`y2 zpt22{QB<}Qp`=0^~*sk;4POqCGck#_PPt&U~KJ|*b--{c6hmy!m+V5Dy7ny_y~Xt zq9W5Zl__yuEC^03H}HTj`;5+oAE35~s_Uz`5n!Ct4kP{AW&K^j@ZCbkerY6(NK81%V!iOem0Cmt~z)2&hxd6L1$I*x*64IzsB z=LZNLKK$h1w$s$#o{lQkp zOn1ahIEf^hXBxArbT315Zj?0Fnwrpp_g=S!7IX*EUAA+L8q!f48O+&B9S@l(=I=X7 z_zMk7%>#2cI~r*E60K$zt|qqu7({=i(h|Dl)mu>g1UWfzq!PTv2>8HC(Av?31|jLz zPRt0a%PpN+#W_>jxv=F%u&jEp>WmcKZ_&wY71l!}EwE^^CQrf4G*;oEYCcu-ySpPw zFf(QUkb4N_01MaJFb87Z&5~2DV2C-;$mp zni-zf9YMv8ZvEWisIt9eTetw9yeI2v&N?h8c*yg4DKA}##EuNFkDyM_E~_Wf5LXOCtTy1Hwe-o{fAGG z=g{oMa7!ZXmA%ywR7>pmzI!gjkckKyybCd3r2K%xE}EL1U2Liusw+K%__}Y(jgg4Z zINvz<)3Ho*VnEGhLSeYkX6LG9H1&bc(I&m$+s5jYqKX^<+fgLa2z z12u7pV8k0~ps>0XsXrklAPajMmt8LJD~J$WR7|KE6-V3A^aL$%7QRqgrq9Ap;WWwv z>L*Y|7+(bqCJAxBO{$D@Dgj*xjm$_#CWtQLx6T>);GB%lE_CqP90gKenAZz?pCj@z zUQthVvlS@@rUpJ8;WSFlju3%9C%Dzq%3mSu)yb1hE!yq4SLnBJV!ZE`3u+!Ki;ou( z&WPXhsaO0BevP|^r-XvxO@G5cb!4u!5n*bWTdnA%_0uAmb96xl9`3-7{B5dg5jqN4-42V0%xiq-yL^oVk7^UbjDr`JSzOceDQvX!(P?9Y3Q)m15NVl2tjRwLc{!kY zy>yr*xdH3K!4?W2f3t_o^o_j%)!EP(OJWQ>S&ElF@cnB7G zzQWFz*Om=RQ#HkIW!-)%J*~!2g`na5NDrv@)b?OBJ4)C<(+q`+j-?ntP~@nCgA7w> zV&f7kRXaa3!vFHF=z~MZfXH`PbuxUI%Z-L^)OJ*|hi3fux z=W#t7-sY;?;_tk4>Eg($_4tu{By`FPem-LGnz8-)Q)wKTNHRTleO6Omam7c#N7L2I zmlQ!&bB@jKDDaqZDg!!B$o9b6cI%WlMljBsGN0}!NvTQ_&I%F8%}US&eKyt!`eG#HCSjoRd=F;8 zc-l!KVHf6Sa$$Md>iyld6HN=i(;@pXk+V79J_~@Fj*hqCAp9m&E2UK5VyQ|o6qC|~ z7jbLKwcHtB-FLODJAl+XY`csG(RxwQQOzsEYSUIzCW;*(Y{&`=+waIN;CcJBTGOh8!MJhmL&A3srQ8w%st#QoimgJPP`vJt!LeVi&uTp83le9C#cL(<#!dFHp2BWtzT==TH2Fc4IEIeq`k*T=PH^ zUM#K&)T%EHRH^TvK^Q$UvE|(omxM6?0pp&k^k$AO*mUtcNjNoxny3pIAIb%8=|(#r zZs;T%=6tCt zAK5Uz(}jfj41>zj)Jy-W+U_!I{|heWnJPQ+bGs$K$K*K|E$KjLG7*c2t{=65m)N5olAFV5GLWzx@p&&ybfKUpj*6GhRj~g4xA+6n4{I zQoeTk)%H5|!!N#iV!uJ}Dr46p{+le*P1#9OGh)2RzDH%cD#v$c0yg)VCg+ApCmk+R zQ1Vf$_qHB?O7aYyvbmms3*GLv$;m)Lqo)}8ZV`!^u-KA^h~Ox7`(U~!5G-F|3_F** z8B~u%D%PL^rS^aX>bC&*T;g>Kk%|mfxQ&6o)lh9Pt2&&m$owHg zLm5BkqM(QOcMjh-^FkQ2~y+RvVmzEZJ&S@*VK`<|;6=d0Oq`~cc* zJhvwfFR0367^4)uifqAN*}fqzD&QRg6ds*hdLK#A$`|xSxR$d5z5&XQ_haCh5XP^k zAqXsjJnUg6Q$aF%%8`D1XTo)r?@X|(_98M}CZc%Qi^I!;g(Bs6AL#*MMkB4On5H2h zQ<)~ODq3Ns_N+)I7j67YwK|4%V{WSFXZPHVwT1=Lb)t&~MG33(5B8ml_qXjY8v!$V zle|K0JE*J!9gOFtBuBOWv2`Y!y+i25w68?PX)UnTdmH9@^qL1j4ub~2dDVKpTG0-( zY@kAoGOE~__abOAJk*CHve2l6{H8IIV{g^(&9gY!=~8iut0wz(=WP;czAW70@B05vEVwn!!8w?e~d)ZqA70ujOYjka~$d8 z_f<`HScasynp<2u1SnxE_FY@{NGIOYxWHH*Odi$gkMm>cPS6q>!vss!@>R&^G%%|c z14jaMBq1%;+T`!C9bVZdL2B28Hzx#fCd|aO!(@7lR~h6s62Wd77@|CW?gSl693kEj zoGV4LN&R6&(b8ky6{ut+9Pj^|ea~6h0ZD&teRXt)1{a^mWBW|@olZ9DWk>&|2)8Gg zTwAVx5ITMnj%^$N0iV*v?oCpXq!tACko)84~F&E0Zx$Q92gH7v?o5kj& z>I|_VaaS`7x1U!3b)YH0tVacM4q6n>NCc=d*)YPgd@U$|-B&Llkb1!>myqpcpXNp`_YJA=34K-i0 z##2h0B0UB6+>YH9Owm+d4a-ARS9)qDj}P@FKn;>`rt^Eb%F)T}+*Zp1T(Gu99$i$G zEbSzpm{uCbOMYcUyH`N2q-t&~>oz;tTE;E8N40Ovu5^G_BuYZ3gd8pXY%evx8iRSz zKEkYvO&JHCVjcK#r!mDfuLXu7Ldu{AU7JC!MVg|mEk(uE?8s%ujo9)P=q&i>9W?)J z8%!|nS3bxp|EU1ig&6LO^Aj&t22{vO$Du+8;^G`=&(s|=cboj*iXmX*)S~ zU}o!^3HVqcqL|YV#*(Cz2ZHf^;Uk^gL7QtFP;utDtfYs;nBK}(u+&2COwdpBhK`Ix zrbxJCU;9_6 z-aYF1Q?__mQ>q`*qkuk+BZJH?lQ!vwg-1sM5V6sK}aoQ^ylil3Zuf=;4B=li_Y~;i3f!TijK==u+pv zXNt_g({X7*jJ`i!^c?G)o;VP6#^<@beZNmNGi!lSGQqDX^N4yG3T4xC$DTRnWoIV4 zDU$AsplW8iRtsuQjr_RuS|vYmXIPs0dWO3&hO9q7-M7uug3}YKvF8Y_zd>rkR1m2) zWWAGO6KlTKv3~RBR{{e`@SVZPFtwh^1;PQ+L*6-&rpibnZis78goEdD2tU?HpyNLC zi3BF5IDLUk=JRBue59miK{E+AjGEOKkSV==c{PC%UpGbia(8lPM-D(K=3v)1D}Kje zH|4xg?Y_XfF-k?5lp%3nvcCkqq-PeZHI`0D>A8VPG^}^MqS=sSh>VP`Iqy2yiz9Tr z)DEcVWQ*Nj*k9K}u|$cbGJ_(&j;p%%#i9G5Jt{LZ#b}rPb7t4uBnS2gr6ZHgMwqAy z45PDQVvaC}cI)a2{*K9@r1Z~$kwm(L$CY6B&#^`0>EU2d z#^!F^bSFn}Rlr7^Sq#|y*{&o%tULU8$N|o7_5U3?C4MxK#D=KYE-=2SGXbxs)iX8K zR8_r>uX{%lwVx@=h)t!oz~m>&x+&>?2qJi3q$>O=MC!b<=ph@{t)BN)H6d*2}mnPkq3e>g(5|Y(CR$-K|w6%XYj!yI2bW8n(f$98vca)70G$+r*_( zmtLJ+Q_;hw->S_eMKcl#A^~F^hL0qhzUAmJ1m@*-%+QzMNwC;QP zp5UUh} zE7u_YiTHmN__XqMs}M1oqsDa*t@q4<6@JkM@|(BK1j>>A^+q3@O1|uMUo?jN>Sw1a z|09&5oZruYsAjxd4t@X|H(V$gsgw>VryRfr>yh*3ZyupH1yF7rh` StepParameter(None, Some(true), None, None, + None, None, Some("The FileConnector to use to create the FileManager implementation")))) + def getFileManager(fileConnector: FileConnector, pipelineContext: PipelineContext): FileManager = + fileConnector.getFileManager(pipelineContext) } case class CopyResults(success: Boolean, fileSize: Long, durationMS: Long, startTime: Date, endTime: Date) diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSConnector.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSConnector.scala new file mode 100644 index 00000000..fa65cb2a --- /dev/null +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSConnector.scala @@ -0,0 +1,11 @@ +package com.acxiom.gcp.pipeline.connectors + +import com.acxiom.gcp.pipeline.GCPCredential +import com.acxiom.pipeline.PipelineContext +import com.acxiom.pipeline.connectors.Connector + +trait GCSConnector extends Connector { + override protected def getCredential(pipelineContext: PipelineContext): Option[GCPCredential] = { + super.getCredential(pipelineContext).asInstanceOf[Option[GCPCredential]] + } +} diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala index 6f2582d4..9a304081 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala @@ -1,7 +1,6 @@ package com.acxiom.gcp.pipeline.connectors import com.acxiom.gcp.fs.GCSFileManager -import com.acxiom.gcp.pipeline.GCPCredential import com.acxiom.gcp.utils.GCPUtilities import com.acxiom.pipeline.connectors.{BatchDataConnector, DataConnectorUtilities} import com.acxiom.pipeline.steps.{DataFrameReaderOptions, DataFrameWriterOptions} @@ -13,7 +12,8 @@ case class GCSDataConnector(override val name: String, override val credentialName: Option[String], override val credential: Option[Credential], override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), - override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) extends BatchDataConnector { + override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) + extends BatchDataConnector with GCSConnector { override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { val path = source.getOrElse("") setSecurity(pipelineContext) @@ -38,11 +38,7 @@ case class GCSDataConnector(override val name: String, } private def setSecurity(pipelineContext: PipelineContext): Unit = { - val finalCredential = (if (credentialName.isDefined) { - pipelineContext.credentialProvider.get.getNamedCredential(credentialName.get) - } else { - credential - }).asInstanceOf[Option[GCPCredential]] + val finalCredential = getCredential(pipelineContext) if (finalCredential.isDefined) { GCPUtilities.setGCSAuthorization(finalCredential.get.authKey, pipelineContext) } diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSFileConnector.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSFileConnector.scala new file mode 100644 index 00000000..88de43eb --- /dev/null +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSFileConnector.scala @@ -0,0 +1,38 @@ +package com.acxiom.gcp.pipeline.connectors + +import com.acxiom.gcp.fs.GCSFileManager +import com.acxiom.gcp.utils.GCPUtilities +import com.acxiom.pipeline.connectors.FileConnector +import com.acxiom.pipeline.fs.FileManager +import com.acxiom.pipeline.{Credential, PipelineContext} + +/** + * Provides an implementation of FileConnector that works with GCS. + * + * @param projectId The project id of the GCS project + * @param bucket The name of the GCS bucket + * @param name The name of this connector + * @param credentialName The optional name of the credential to provide the CredentialProvider + * @param credential The optional credential to use. credentialName takes precedence if provided. + */ +case class GCSFileConnector(projectId: String, + bucket: String, + override val name: String, + override val credentialName: Option[String], + override val credential: Option[Credential]) extends FileConnector with GCSConnector { + /** + * Creates and opens a FileManager. + * + * @param pipelineContext The current PipelineContext for this session. + * @return A FileManager for this specific connector type + */ + override def getFileManager(pipelineContext: PipelineContext): FileManager = { + val finalCredential = getCredential(pipelineContext) + val jsonAuth = if (finalCredential.isDefined) { + Some(new String(GCPUtilities.generateCredentialsByteArray(Some(finalCredential.get.authKey)).get)) + } else { + None + } + new GCSFileManager(projectId, bucket, jsonAuth) + } +} From dcc11ae08f429a62baa1762b1c26e6ecf91be8a1 Mon Sep 17 00:00:00 2001 From: dafreels Date: Thu, 16 Sep 2021 13:56:41 -0400 Subject: [PATCH 13/24] #252 Added tests for FileConnectors --- .../steps/FileManagerStepsTests.scala | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/FileManagerStepsTests.scala b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/FileManagerStepsTests.scala index b1f6976b..43921dc4 100644 --- a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/FileManagerStepsTests.scala +++ b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/FileManagerStepsTests.scala @@ -1,9 +1,7 @@ package com.acxiom.pipeline.steps -import java.io.File -import java.nio.file.{Files, Path, StandardCopyOption} - import com.acxiom.pipeline._ +import com.acxiom.pipeline.connectors.{HDFSFileConnector, SFTPFileConnector} import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.FileSystem import org.apache.hadoop.hdfs.{HdfsConfiguration, MiniDFSCluster} @@ -13,6 +11,9 @@ import org.apache.spark.sql.SparkSession import org.scalatest.{BeforeAndAfterAll, FunSpec} import software.sham.sftp.MockSftpServer +import java.io.File +import java.nio.file.{Files, Path, StandardCopyOption} + class FileManagerStepsTests extends FunSpec with BeforeAndAfterAll { val MASTER = "local[2]" val APPNAME = "file-manager-steps-spark" @@ -70,22 +71,23 @@ class FileManagerStepsTests extends FunSpec with BeforeAndAfterAll { describe("FileManagerSteps - Copy") { it("Should fail when strict host checking is enabled against localhost") { - val sftp = SFTPSteps.createFileManager("localhost", Some("tester"), Some("testing"), Some(SFTP_PORT), Some(true), pipelineContext) - assert(sftp.isDefined) + val sftpConnector = SFTPFileConnector("localhost", "sftp-connector", None, + Some(UserNameCredential(Map("username" -> "tester", "password" -> "testing"))), + Some(SFTP_PORT), None, None, Some(Map[String, String]("StrictHostKeyChecking" -> "yes"))) val exception = intercept[com.jcraft.jsch.JSchException] { - sftp.get.connect() + sftpConnector.getFileManager(pipelineContext) } assert(Option(exception).nonEmpty) assert(exception.getMessage == "reject HostKey: localhost") } it("Should copy from/to SFTP to HDFS") { - val hdfs = HDFSSteps.createFileManager(pipelineContext) + val hdfsConnector = HDFSFileConnector("my-connector", None, None) + val hdfs = hdfsConnector.getFileManager(pipelineContext) val sftp = SFTPSteps.createFileManager("localhost", Some("tester"), Some("testing"), Some(SFTP_PORT), Some(false), pipelineContext) - assert(hdfs.isDefined) assert(sftp.isDefined) // Verify that the HDFS file system has nothing - assert(hdfs.get.getFileListing("/").isEmpty) + assert(hdfs.getFileListing("/").isEmpty) // Connect to the SFTP file system sftp.get.connect() @@ -96,13 +98,13 @@ class FileManagerStepsTests extends FunSpec with BeforeAndAfterAll { assert(originalSftpFile.isDefined) // Copy from SFTP to HDFS - FileManagerSteps.copy(sftp.get, "/MOCK_DATA.csv", hdfs.get, "/COPIED_DATA.csv") - val copiedHdfsFile = hdfs.get.getFileListing("/").find(_.fileName == "COPIED_DATA.csv") + FileManagerSteps.copy(sftp.get, "/MOCK_DATA.csv", hdfs, "/COPIED_DATA.csv") + val copiedHdfsFile = hdfs.getFileListing("/").find(_.fileName == "COPIED_DATA.csv") assert(copiedHdfsFile.isDefined) assert(originalSftpFile.get.size == copiedHdfsFile.get.size) // Copy from HDFS to SFTP - FileManagerSteps.copy(hdfs.get, "/COPIED_DATA.csv", sftp.get, "/HDFS_COPIED_DATA.csv") + FileManagerSteps.copy(hdfs, "/COPIED_DATA.csv", sftp.get, "/HDFS_COPIED_DATA.csv") val sftpCopiedHdfsFile = sftp.get.getFileListing("/").find(_.fileName == "HDFS_COPIED_DATA.csv") assert(sftpCopiedHdfsFile.isDefined) assert(originalSftpFile.get.size == sftpCopiedHdfsFile.get.size) From 2e51fc0d98369932b2400eef9dfd953d25e7e77a Mon Sep 17 00:00:00 2001 From: dafreels Date: Fri, 17 Sep 2021 11:52:44 -0400 Subject: [PATCH 14/24] #252 Moved the read and write options from the constructor to the functions within a DataConnector. --- docs/dataconnectors.md | 20 ++++----- docs/fileconnectors.md | 8 ++-- .../connectors/KinesisDataConnector.scala | 15 +++---- .../pipeline/connectors/S3DataConnector.scala | 13 +++--- .../pipeline/connectors/DataConnector.scala | 11 +++-- .../connectors/HDFSDataConnector.scala | 13 +++--- .../pipeline/steps/DataConnectorSteps.scala | 12 ++++-- .../pipeline/steps/FileManagerSteps.scala | 2 +- .../steps/DataConnectorStepsTests.scala | 41 ++++++++----------- .../connectors/BigQueryDataConnector.scala | 12 +++--- .../connectors/GCSDataConnector.scala | 13 +++--- .../com/acxiom/gcp/steps/BigQuerySteps.scala | 8 ++-- .../connectors/KafkaDataConnector.scala | 11 +++-- .../connectors/MongoDataConnector.scala | 13 +++--- 14 files changed, 104 insertions(+), 88 deletions(-) diff --git a/docs/dataconnectors.md b/docs/dataconnectors.md index dd6c9cea..95577388 100644 --- a/docs/dataconnectors.md +++ b/docs/dataconnectors.md @@ -11,8 +11,6 @@ The following parameters are available to all data connectors: * **name** - The name of the connector * **credentialName** - The optional credential name to use to authenticate * **credential** - The optional credential to use to authenticate -* **readOptions** - The optional read options to use when loading the DataFrame -* **writeOptions** - The optional write options to use when writing the DataFrame ## Batch Connectors that are designed to load and write data for batch processing will extend the _BatchDataConnector_. These @@ -31,7 +29,7 @@ val connector = HDFSDataConnector("my-connector", None, None, #### Globals JSON ```json { - "connector": { + "myConnector": { "className": "com.acxiom.pipeline.connectors.HDFSDataConnector", "object": { "name": "my-connector", @@ -59,7 +57,7 @@ val connector = S3DataConnector("my-connector", Some("my-credential-name-for-sec #### Globals JSON ```json { - "connector": { + "myS3Connector": { "className": "com.acxiom.aws.pipeline.connectors.S3DataConnector", "object": { "name": "my-connector", @@ -88,7 +86,7 @@ val connector = GCSDataConnector("my-connector", Some("my-credential-name-for-se #### Globals JSON ```json { - "connector": { + "myGCSConnector": { "className": "com.acxiom.gcp.pipeline.connectors.GCSDataConnector", "object": { "name": "my-connector", @@ -117,7 +115,7 @@ val connector = BigQueryDataConnector("temp-bucket-name", "my-connector", Some(" #### Globals JSON ```json { - "connector": { + "bigQueryConnector": { "className": "com.acxiom.gcp.pipeline.connectors.BigQueryDataConnector", "object": { "name": "my-connector", @@ -136,14 +134,12 @@ the standard parameters, the following parameters are available: #### Scala ```scala -val connector = MongoDataConnector("mongodb://127.0.0.1/test", "myCollectionName", "my-connector", Some("my-credential-name-for-secrets-manager"), None, - DataFrameReaderOptions(format = "csv"), - DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) +val connector = MongoDataConnector("mongodb://127.0.0.1/test", "myCollectionName", "my-connector", Some("my-credential-name-for-secrets-manager"), None) ``` #### Globals JSON ```json { - "connector": { + "customMongoConnector": { "className": "com.acxiom.metalus.pipeline.connectors.MongoDataConnector", "object": { "name": "my-connector", @@ -178,7 +174,7 @@ val connector = KinesisDataConnector("stream-name", "us-east-1", None, Some(15), #### Globals JSON ```json { - "connector": { + "kinesisConnector": { "className": "com.acxiom.aws.pipeline.connectors.KinesisDataConnector", "object": { "name": "my-connector", @@ -209,7 +205,7 @@ val connector = KafkaDataConnector("topic-name1,topic-name2", "host1:port1,host2 #### Globals JSON ```json { - "connector": { + "kafkaConnector": { "className": "com.acxiom.kafka.pipeline.connectors.KafkaDataConnector", "object": { "name": "my-connector", diff --git a/docs/fileconnectors.md b/docs/fileconnectors.md index 31c1f374..a369fa78 100644 --- a/docs/fileconnectors.md +++ b/docs/fileconnectors.md @@ -18,12 +18,12 @@ This connector provides access to the HDFS file system. The _credentialName_ and this implementation, instead relying on the permissions of the cluster. Below is an example setup: #### Scala ```scala -val connector = HDFSFileConnector("my-connector", None, None) +val connector = HDFSFileConnector("my-hdfs-connector", None, None) ``` #### Globals JSON ```json { - "connector": { + "myHdfsConnector": { "className": "com.acxiom.pipeline.connectors.HDFSFileConnector", "object": { "name": "my-connector" @@ -45,12 +45,12 @@ available: Below is an example setup: #### Scala ```scala -val connector = SFTPFileConnector("sftp.myhost.com", "my-connector", None, None) +val connector = SFTPFileConnector("sftp.myhost.com", "my-sftp-connector", None, None) ``` #### Globals JSON ```json { - "connector": { + "sftpConnector": { "className": "com.acxiom.pipeline.connectors.SFTPFileConnector", "object": { "name": "my-connector", diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisDataConnector.scala b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisDataConnector.scala index b096669d..a688ff58 100644 --- a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisDataConnector.scala +++ b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/KinesisDataConnector.scala @@ -19,8 +19,6 @@ import org.apache.spark.sql.streaming.StreamingQuery * @param name The name of the connector * @param credentialName The optional name of the credential to use when authorizing to the Kinesis stream * @param credential The optional credential to use when authorizing to the Kinesis stream - * @param readOptions The optional read options to use - * @param writeOptions The optional write options to use */ case class KinesisDataConnector(streamName: String, region: String = "us-east-1", @@ -29,11 +27,11 @@ case class KinesisDataConnector(streamName: String, separator: String = ",", override val name: String, override val credentialName: Option[String], - override val credential: Option[Credential], - override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), - override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) + override val credential: Option[Credential]) extends StreamingDataConnector with AWSConnector { - override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { + override def load(source: Option[String], + pipelineContext: PipelineContext, + readOptions: DataFrameReaderOptions = DataFrameReaderOptions()): DataFrame = { val initialReader = pipelineContext.sparkSession.get.readStream .format("kinesis") .option("streamName", streamName) @@ -73,7 +71,10 @@ case class KinesisDataConnector(streamName: String, finalReader.load() } - override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { + override def write(dataFrame: DataFrame, + destination: Option[String], + pipelineContext: PipelineContext, + writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()): Option[StreamingQuery] = { val finalCredential: Option[AWSCredential] = getCredential(pipelineContext) if (dataFrame.isStreaming) { Some(dataFrame diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3DataConnector.scala b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3DataConnector.scala index 08883928..5e7ec6c1 100644 --- a/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3DataConnector.scala +++ b/metalus-aws/src/main/scala/com/acxiom/aws/pipeline/connectors/S3DataConnector.scala @@ -9,11 +9,11 @@ import org.apache.spark.sql.streaming.StreamingQuery case class S3DataConnector(override val name: String, override val credentialName: Option[String], - override val credential: Option[Credential], - override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), - override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) + override val credential: Option[Credential]) extends BatchDataConnector with AWSConnector { - override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { + override def load(source: Option[String], + pipelineContext: PipelineContext, + readOptions: DataFrameReaderOptions = DataFrameReaderOptions()): DataFrame = { val path = source.getOrElse("") setSecurity(pipelineContext, path) @@ -21,7 +21,10 @@ case class S3DataConnector(override val name: String, .load(S3Utilities.replaceProtocol(path, S3Utilities.deriveProtocol(path))) } - override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { + override def write(dataFrame: DataFrame, + destination: Option[String], + pipelineContext: PipelineContext, + writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()): Option[StreamingQuery] = { val path = destination.getOrElse("") setSecurity(pipelineContext, path) if (dataFrame.isStreaming) { diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala index 5976885c..938f1865 100644 --- a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/DataConnector.scala @@ -6,10 +6,13 @@ import org.apache.spark.sql.DataFrame import org.apache.spark.sql.streaming.StreamingQuery trait DataConnector extends Connector { - def readOptions: DataFrameReaderOptions = DataFrameReaderOptions() - def writeOptions: DataFrameWriterOptions = DataFrameWriterOptions() - def load(source: Option[String], pipelineContext: PipelineContext): DataFrame - def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] + def load(source: Option[String], + pipelineContext: PipelineContext, + readOptions: DataFrameReaderOptions = DataFrameReaderOptions()): DataFrame + def write(dataFrame: DataFrame, + destination: Option[String], + pipelineContext: PipelineContext, + writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()): Option[StreamingQuery] } trait BatchDataConnector extends DataConnector {} diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSDataConnector.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSDataConnector.scala index 8b2052fe..3805b0e0 100644 --- a/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSDataConnector.scala +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/connectors/HDFSDataConnector.scala @@ -7,14 +7,17 @@ import org.apache.spark.sql.streaming.StreamingQuery case class HDFSDataConnector(override val name: String, override val credentialName: Option[String], - override val credential: Option[Credential], - override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), - override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) extends BatchDataConnector { + override val credential: Option[Credential]) extends BatchDataConnector { - override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = + override def load(source: Option[String], + pipelineContext: PipelineContext, + readOptions: DataFrameReaderOptions = DataFrameReaderOptions()): DataFrame = DataConnectorUtilities.buildDataFrameReader(pipelineContext.sparkSession.get, readOptions).load(source.getOrElse("")) - override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { + override def write(dataFrame: DataFrame, + destination: Option[String], + pipelineContext: PipelineContext, + writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()): Option[StreamingQuery] = { if (dataFrame.isStreaming) { Some(dataFrame.writeStream .format(writeOptions.format) diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataConnectorSteps.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataConnectorSteps.scala index 8a4e6820..df1b7e1a 100644 --- a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataConnectorSteps.scala +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/DataConnectorSteps.scala @@ -16,11 +16,13 @@ object DataConnectorSteps { "Connectors") @StepParameters(Map("dataFrame" -> StepParameter(None, Some(true), None, None, None, None, Some("The DataFrame to write")), "connector" -> StepParameter(None, Some(true), None, None, None, None, Some("The data connector to use when writing")), - "source" -> StepParameter(None, Some(false), None, None, None, None, Some("The source path to load data")))) + "source" -> StepParameter(None, Some(false), None, None, None, None, Some("The source path to load data")), + "readOptions" -> StepParameter(None, Some(false), None, None, None, None, Some("The optional options to use while reading the data")))) def loadDataFrame(connector: DataConnector, source: Option[String], + readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), pipelineContext: PipelineContext): DataFrame = - connector.load(source, pipelineContext) + connector.load(source, pipelineContext, readOptions) @StepFunction("5608eba7-e9ff-48e6-af77-b5e810b99d89", "Write", @@ -29,10 +31,12 @@ object DataConnectorSteps { "Connectors") @StepParameters(Map("dataFrame" -> StepParameter(None, Some(true), None, None, None, None, Some("The DataFrame to write")), "connector" -> StepParameter(None, Some(true), None, None, None, None, Some("The data connector to use when writing")), - "destination" -> StepParameter(None, Some(false), None, None, None, None, Some("The destination path to write data")))) + "destination" -> StepParameter(None, Some(false), None, None, None, None, Some("The destination path to write data")), + "writeOptions" -> StepParameter(None, Some(false), None, None, None, None, Some("The optional DataFrame options to use while writing")))) def writeDataFrame(dataFrame: DataFrame, connector: DataConnector, destination: Option[String], + writeOptions: DataFrameWriterOptions = DataFrameWriterOptions(), pipelineContext: PipelineContext): Option[StreamingQuery] = - connector.write(dataFrame, destination, pipelineContext) + connector.write(dataFrame, destination, pipelineContext, writeOptions) } diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/FileManagerSteps.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/FileManagerSteps.scala index 5ca2e312..8009948c 100644 --- a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/FileManagerSteps.scala +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/FileManagerSteps.scala @@ -143,7 +143,7 @@ object FileManagerSteps { "Create a FileManager", "Creates a FileManager using the provided FileConnector", "Pipeline", - "InputOutput") + "Connectors") @StepParameters(Map("fileConnector" -> StepParameter(None, Some(true), None, None, None, None, Some("The FileConnector to use to create the FileManager implementation")))) def getFileManager(fileConnector: FileConnector, pipelineContext: PipelineContext): FileManager = diff --git a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/DataConnectorStepsTests.scala b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/DataConnectorStepsTests.scala index c84955ff..c3e53117 100644 --- a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/DataConnectorStepsTests.scala +++ b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/DataConnectorStepsTests.scala @@ -81,10 +81,9 @@ class DataConnectorStepsTests extends FunSpec with BeforeAndAfterAll { val dataFrame = chickens.toDF("id", "chicken") - val connector = HDFSDataConnector("my-connector", None, None, - DataFrameReaderOptions(), - DataFrameWriterOptions(format = "csv")) - DataConnectorSteps.writeDataFrame(dataFrame, connector, Some(miniCluster.getURI + "/data/chickens.csv"), pipelineContext) + val connector = HDFSDataConnector("my-connector", None, None) + DataConnectorSteps.writeDataFrame(dataFrame, connector, Some(miniCluster.getURI + "/data/chickens.csv"), + DataFrameWriterOptions(format = "csv"), pipelineContext) val list = readHDFSContent(fs, miniCluster.getURI + "/data/chickens.csv") assert(list.size == 3) @@ -105,10 +104,9 @@ class DataConnectorStepsTests extends FunSpec with BeforeAndAfterAll { val dataFrame = chickens.toDF("id", "chicken") DataFrameSteps.persistDataFrame(dataFrame, "MEMORY_ONLY") - val connector = HDFSDataConnector("my-connector", None, None, - DataFrameReaderOptions(), - DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) - DataConnectorSteps.writeDataFrame(dataFrame, connector, Some(miniCluster.getURI + "/data/chickens.csv"), pipelineContext) + val connector = HDFSDataConnector("my-connector", None, None) + DataConnectorSteps.writeDataFrame(dataFrame, connector, Some(miniCluster.getURI + "/data/chickens.csv"), + DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ"))), pipelineContext) DataFrameSteps.unpersistDataFrame(dataFrame) val list = readHDFSContent(fs, miniCluster.getURI + "/data/chickens.csv") @@ -131,10 +129,9 @@ class DataConnectorStepsTests extends FunSpec with BeforeAndAfterAll { val dataFrame = chickens.toDF("id", "chicken") val path = miniCluster.getURI + "/data/chickens.csv" - val connector = HDFSDataConnector("my-connector", None, None, - DataFrameReaderOptions(), - DataFrameWriterOptions(format = "csv", saveMode = "overwrite")) - DataConnectorSteps.writeDataFrame(dataFrame, connector, Some(path), pipelineContext) + val connector = HDFSDataConnector("my-connector", None, None) + DataConnectorSteps.writeDataFrame(dataFrame, connector, Some(path), + DataFrameWriterOptions(format = "csv", saveMode = "overwrite"), pipelineContext) val list = readHDFSContent(fs, miniCluster.getURI + "/data/chickens.csv") assert(list.size == 3) var writtenData: Seq[(String, String)] = Seq() @@ -159,9 +156,9 @@ class DataConnectorStepsTests extends FunSpec with BeforeAndAfterAll { val path = miniCluster.getURI + "/data/chickens2.csv" writeHDFSContext(fs, path, csv) - val connector = HDFSDataConnector("my-connector", None, None, - DataFrameReaderOptions(format = "csv")) - val dataFrame = DataConnectorSteps.loadDataFrame(connector, Some(path), pipelineContext) + val connector = HDFSDataConnector("my-connector", None, None) + val dataFrame = DataConnectorSteps.loadDataFrame(connector, Some(path), + DataFrameReaderOptions(format = "csv"), pipelineContext) DataFrameSteps.persistDataFrame(dataFrame) assert(dataFrame.count() == 3) val result = dataFrame.take(3).map(r => (r.getString(0), r.getString(1))).toSeq @@ -174,10 +171,9 @@ class DataConnectorStepsTests extends FunSpec with BeforeAndAfterAll { val path = miniCluster.getURI + "/data/chickens2.csv" writeHDFSContext(fs, path, csv) - val connector = HDFSDataConnector("my-connector", None, None, - DataFrameReaderOptions(format = "csv", options = Some(Map("header" -> "true", "delimiter" -> "þ"))), - DataFrameWriterOptions()) - val dataFrame = DataConnectorSteps.loadDataFrame(connector, Some(path), pipelineContext) + val connector = HDFSDataConnector("my-connector", None, None) + val dataFrame = DataConnectorSteps.loadDataFrame(connector, Some(path), + DataFrameReaderOptions(format = "csv", options = Some(Map("header" -> "true", "delimiter" -> "þ"))), pipelineContext) assert(dataFrame.count() == 3) val result = dataFrame.take(3).map(r => (r.getString(0), r.getString(1))).toSeq @@ -188,10 +184,9 @@ class DataConnectorStepsTests extends FunSpec with BeforeAndAfterAll { val csv = "1,silkie\n2,polish\n3,sultan" val path = miniCluster.getURI + "/data/chickens2.csv" writeHDFSContext(fs, path, csv) - val connector = HDFSDataConnector("my-connector", None, None, - DataFrameReaderOptions(format = "csv"), - DataFrameWriterOptions()) - val dataFrame = DataConnectorSteps.loadDataFrame(connector, Some(path), pipelineContext) + val connector = HDFSDataConnector("my-connector", None, None) + val dataFrame = DataConnectorSteps.loadDataFrame(connector, Some(path), + DataFrameReaderOptions(format = "csv"), pipelineContext) val results = dataFrame.collect().map(r => (r.getString(0), r.getString(1))).toSeq assert(results.size == 3) assert(results == chickens) diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/BigQueryDataConnector.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/BigQueryDataConnector.scala index 88422d5a..9e336568 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/BigQueryDataConnector.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/BigQueryDataConnector.scala @@ -13,10 +13,10 @@ import java.util.Base64 case class BigQueryDataConnector(tempWriteBucket: String, override val name: String, override val credentialName: Option[String], - override val credential: Option[Credential], - override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), - override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) extends BatchDataConnector { - override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { + override val credential: Option[Credential]) extends BatchDataConnector { + override def load(source: Option[String], + pipelineContext: PipelineContext, + readOptions: DataFrameReaderOptions = DataFrameReaderOptions()): DataFrame = { val table = source.getOrElse("") val readerOptions = readOptions.copy(format = "bigquery") // Setup authentication @@ -29,7 +29,9 @@ case class BigQueryDataConnector(tempWriteBucket: String, DataConnectorUtilities.buildDataFrameReader(pipelineContext.sparkSession.get, finalOptions).load(table) } - override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { + override def write(dataFrame: DataFrame, destination: Option[String], + pipelineContext: PipelineContext, + writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()): Option[StreamingQuery] = { val table = destination.getOrElse("") // Setup format for BigQuery val writerOptions = writeOptions.copy(format = "bigquery", options = Some(Map("temporaryGcsBucket" -> tempWriteBucket))) diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala index 9a304081..303f48ea 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/pipeline/connectors/GCSDataConnector.scala @@ -10,18 +10,21 @@ import org.apache.spark.sql.streaming.StreamingQuery case class GCSDataConnector(override val name: String, override val credentialName: Option[String], - override val credential: Option[Credential], - override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), - override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) + override val credential: Option[Credential]) extends BatchDataConnector with GCSConnector { - override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { + override def load(source: Option[String], + pipelineContext: PipelineContext, + readOptions: DataFrameReaderOptions = DataFrameReaderOptions()): DataFrame = { val path = source.getOrElse("") setSecurity(pipelineContext) DataConnectorUtilities.buildDataFrameReader(pipelineContext.sparkSession.get, readOptions) .load(GCSFileManager.prepareGCSFilePath(path)) } - override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { + override def write(dataFrame: DataFrame, + destination: Option[String], + pipelineContext: PipelineContext, + writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()): Option[StreamingQuery] = { val path = destination.getOrElse("") setSecurity(pipelineContext) if (dataFrame.isStreaming) { diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/BigQuerySteps.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/BigQuerySteps.scala index 0d51922b..df3b8599 100644 --- a/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/BigQuerySteps.scala +++ b/metalus-gcp/src/main/scala/com/acxiom/gcp/steps/BigQuerySteps.scala @@ -33,8 +33,8 @@ object BigQuerySteps { } else { None } - val connector = BigQueryDataConnector("", "readFromTable", None, creds, readerOptions) - connector.load(Some(table), pipelineContext) + val connector = BigQueryDataConnector("", "readFromTable", None, creds) + connector.load(Some(table), pipelineContext, readerOptions) } @StepFunction("5b6e114b-51bb-406f-a95a-2a07bc0d05c7", @@ -64,7 +64,7 @@ object BigQuerySteps { } else { None } - val connector = BigQueryDataConnector(tempBucket, "writeToTable", None, creds, DataFrameReaderOptions(), writerOptions) - connector.write(dataFrame, Some(table), pipelineContext) + val connector = BigQueryDataConnector(tempBucket, "writeToTable", None, creds) + connector.write(dataFrame, Some(table), pipelineContext, writerOptions) } } diff --git a/metalus-kafka/src/main/scala/com/acxiom/kafka/pipeline/connectors/KafkaDataConnector.scala b/metalus-kafka/src/main/scala/com/acxiom/kafka/pipeline/connectors/KafkaDataConnector.scala index 48a2ff48..82236c21 100644 --- a/metalus-kafka/src/main/scala/com/acxiom/kafka/pipeline/connectors/KafkaDataConnector.scala +++ b/metalus-kafka/src/main/scala/com/acxiom/kafka/pipeline/connectors/KafkaDataConnector.scala @@ -15,11 +15,11 @@ case class KafkaDataConnector(topics: String, override val name: String, override val credentialName: Option[String], override val credential: Option[Credential], - override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), - override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions(), clientId: String = "metalus_default_kafka_producer_client", separator: String = ",") extends StreamingDataConnector { - override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { + override def load(source: Option[String], + pipelineContext: PipelineContext, + readOptions: DataFrameReaderOptions = DataFrameReaderOptions()): DataFrame = { pipelineContext.sparkSession.get .readStream .format("kafka") @@ -29,7 +29,10 @@ case class KafkaDataConnector(topics: String, .load() } - override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { + override def write(dataFrame: DataFrame, + destination: Option[String], + pipelineContext: PipelineContext, + writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()): Option[StreamingQuery] = { if (dataFrame.isStreaming) { Some(dataFrame .writeStream diff --git a/metalus-mongo/src/main/scala/com/acxiom/metalus/pipeline/connectors/MongoDataConnector.scala b/metalus-mongo/src/main/scala/com/acxiom/metalus/pipeline/connectors/MongoDataConnector.scala index bc5a17ce..5a2319c4 100644 --- a/metalus-mongo/src/main/scala/com/acxiom/metalus/pipeline/connectors/MongoDataConnector.scala +++ b/metalus-mongo/src/main/scala/com/acxiom/metalus/pipeline/connectors/MongoDataConnector.scala @@ -17,19 +17,22 @@ case class MongoDataConnector(uri: String, collectionName: String, override val name: String, override val credentialName: Option[String], - override val credential: Option[Credential], - override val readOptions: DataFrameReaderOptions = DataFrameReaderOptions(), - override val writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()) extends BatchDataConnector { + override val credential: Option[Credential]) extends BatchDataConnector { private val passwordTest = "[@#?\\/\\[\\]:]".r private val connectionString = new ConnectionString(uri) - override def load(source: Option[String], pipelineContext: PipelineContext): DataFrame = { + override def load(source: Option[String], + pipelineContext: PipelineContext, + readOptions: DataFrameReaderOptions = DataFrameReaderOptions()): DataFrame = { MongoSpark.loadAndInferSchema(pipelineContext.sparkSession.get, ReadConfig(Map("collection" -> collectionName, "uri" -> buildConnectionString(pipelineContext)))) } - override def write(dataFrame: DataFrame, destination: Option[String], pipelineContext: PipelineContext): Option[StreamingQuery] = { + override def write(dataFrame: DataFrame, + destination: Option[String], + pipelineContext: PipelineContext, + writeOptions: DataFrameWriterOptions = DataFrameWriterOptions()): Option[StreamingQuery] = { val writeConfig = WriteConfig(Map("collection" -> collectionName, "uri" -> buildConnectionString(pipelineContext))) if (dataFrame.isStreaming) { Some(dataFrame From e656fb548418c211be2b37893bb550e260b64099 Mon Sep 17 00:00:00 2001 From: dafreels Date: Tue, 5 Oct 2021 07:42:23 -0400 Subject: [PATCH 15/24] #254 Added retries at the step level --- docs/error-handling.md | 5 ++ docs/pipeline-steps.md | 5 ++ .../scala/com/acxiom/pipeline/Pipeline.scala | 9 +++ .../com/acxiom/pipeline/PipelineStep.scala | 24 ++++--- .../acxiom/pipeline/flow/PipelineFlow.scala | 53 ++++++++------ .../com/acxiom/pipeline/PipelineDefs.scala | 12 ++++ .../com/acxiom/pipeline/SparkSuiteTests.scala | 71 ++++++++++++++++++- 7 files changed, 146 insertions(+), 33 deletions(-) diff --git a/docs/error-handling.md b/docs/error-handling.md index a42526e5..d7987bc2 100644 --- a/docs/error-handling.md +++ b/docs/error-handling.md @@ -24,6 +24,11 @@ both execute normally and invoke _Error Step_. The _Error Step_ will throw an e _Error Handler_ step and then complete the pipeline. _Step 4_ will never be executed when there is an exception. Metalus will consider this as a successful execution. +## Step Retry +A step which defines the _retryLimit_ attribute will be automatically restarted when an exception is thrown. Once the +limit has been reached, the [nextStepOnError](#next-step-error-handling) will be invoked or the exception will +be thrown to stop the pipeline. + ![Next Step On Error Flow](images/next_step_on_error_flow.png) ## Pipeline Exceptions Metalus uses the _PipelineStepException_ trait as the base for application exceptions. Any exception that extends diff --git a/docs/pipeline-steps.md b/docs/pipeline-steps.md index a7534e6a..06096790 100644 --- a/docs/pipeline-steps.md +++ b/docs/pipeline-steps.md @@ -46,6 +46,11 @@ The pipeline step will include a new attribute named **nextStepId** which indica that should be executed. Branch steps will not have this attribute since the next step id is determined by the **params** array **result** type parameters. +### retryLimit +The pipeline step has an optional attribute named **retryLimit** which will retry the step if an exception is thrown. +Once the limit has been reached, the **nextStepOnError** will be called if it has been defined or the exception will +be thrown to stop the pipeline. + ### params At a minimum, the parameter from the step template should be replicated here. Required parameters will have to additionally set the **value** attribute unless the **defaultValue** attribute is provided. Additional attributes will be required diff --git a/metalus-core/src/main/scala/com/acxiom/pipeline/Pipeline.scala b/metalus-core/src/main/scala/com/acxiom/pipeline/Pipeline.scala index 9fff557b..672bbaa1 100644 --- a/metalus-core/src/main/scala/com/acxiom/pipeline/Pipeline.scala +++ b/metalus-core/src/main/scala/com/acxiom/pipeline/Pipeline.scala @@ -165,6 +165,15 @@ case class PipelineContext(sparkConf: Option[SparkConf] = None, def setGlobals(globals: Map[String, Any]): PipelineContext = this.copy(globals = Some(if(this.globals.isDefined) this.globals.get ++ globals else globals)) + /** + * This function will remove a single entry on the globals map. + * + * @param globalName The name of the global property to remove. + * @return A new PipelineContext with an updated globals map. + */ + def removeGlobal(globalName: String): PipelineContext = + this.copy(globals = Some(this.globals.getOrElse(Map[String, Any]()) - globalName)) + /** * Adds a new PipelineStepMessage to the context * diff --git a/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineStep.scala b/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineStep.scala index 4762bba1..c09de2cc 100644 --- a/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineStep.scala +++ b/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineStep.scala @@ -7,15 +7,18 @@ import java.util.Date /** * Metadata about the next step in the pipeline process. * - * @param id The unique (to the pipeline) id of this step. This pproperty is used to chain steps together. - * @param displayName A name that can be displayed in logs and errors. - * @param description A long description of this step. - * @param `type` The type of step. - * @param params The step parameters that are used during execution. - * @param engineMeta Contains the instruction for invoking the step function. - * @param executeIfEmpty This field allows a value to be passed in rather than executing the step. - * @param nextStepId The id of the next step to execute. - * @param stepId The id of the step that provided the metadata. + * @param id The unique (to the pipeline) id of this step. This pproperty is used to chain steps together. + * @param displayName A name that can be displayed in logs and errors. + * @param description A long description of this step. + * @param `type` The type of step. + * @param params The step parameters that are used during execution. + * @param engineMeta Contains the instruction for invoking the step function. + * @param executeIfEmpty This field allows a value to be passed in rather than executing the step. + * @param nextStepId The id of the next step to execute. + * @param stepId The id of the step that provided the metadata. + * @param nextStepOnError The id of the step that should be called on error + * @param retryLimit The number of times that this step should be retried on error. Default is -1 indicating to + * not retry. This parameter will be considered before nextStepOnError. */ case class PipelineStep(id: Option[String] = None, displayName: Option[String] = None, @@ -26,7 +29,8 @@ case class PipelineStep(id: Option[String] = None, nextStepId: Option[String] = None, executeIfEmpty: Option[String] = None, stepId: Option[String] = None, - nextStepOnError: Option[String] = None) + nextStepOnError: Option[String] = None, + retryLimit: Option[Int] = Some(-1)) /** * Represents a single parameter in a step. diff --git a/metalus-core/src/main/scala/com/acxiom/pipeline/flow/PipelineFlow.scala b/metalus-core/src/main/scala/com/acxiom/pipeline/flow/PipelineFlow.scala index 05e7a902..46b42e2f 100644 --- a/metalus-core/src/main/scala/com/acxiom/pipeline/flow/PipelineFlow.scala +++ b/metalus-core/src/main/scala/com/acxiom/pipeline/flow/PipelineFlow.scala @@ -151,27 +151,7 @@ trait PipelineFlow { pipelineContext: PipelineContext): PipelineContext = { logger.debug(s"Executing Step (${step.id.getOrElse("")}) ${step.displayName.getOrElse("")}") val ssContext = PipelineFlow.handleEvent(pipelineContext, "pipelineStepStarted", List(pipeline, step, pipelineContext)) - val (nextStepId, sfContext) = try { - val result = processPipelineStep(step, pipeline, ssContext) - // setup the next step - val (newPipelineContext, nextStepId) = result match { - case flowResult: FlowResult => (updatePipelineContext(step, result, flowResult.nextStepId, flowResult.pipelineContext), flowResult.nextStepId) - case _ => - val nextStepId = getNextStepId(step, result) - (updatePipelineContext(step, result, nextStepId, ssContext), nextStepId) - } - // run the step finished event - val sfContext = PipelineFlow.handleEvent(newPipelineContext, "pipelineStepFinished", List(pipeline, step, newPipelineContext)) - (nextStepId, sfContext) - } catch { - case e: Throwable if step.nextStepOnError.isDefined => - // handle exception - val ex = PipelineFlow.handleStepExecutionExceptions(e, pipeline, pipelineContext) - // put exception on the context as the "result" for this step. - val updateContext = updatePipelineContext(step, PipelineStepResponse(Some(ex), None), step.nextStepOnError, ssContext) - (step.nextStepOnError, updateContext) - case e => throw e - } + val (nextStepId, sfContext) = processStepWithRetry(step, pipeline, ssContext, pipelineContext, 0) // Call the next step here if (steps.contains(nextStepId.getOrElse("")) && (steps(nextStepId.getOrElse("")).`type`.getOrElse("") == PipelineStepType.JOIN || @@ -194,6 +174,37 @@ trait PipelineFlow { } } + private def processStepWithRetry(step: PipelineStep, pipeline: Pipeline, + ssContext: PipelineContext, + pipelineContext: PipelineContext, + stepRetryCount: Int): (Option[String], PipelineContext) = { + try { + val result = processPipelineStep(step, pipeline, ssContext.setGlobal("stepRetryCount", stepRetryCount)) + // setup the next step + val (newPipelineContext, nextStepId) = result match { + case flowResult: FlowResult => (updatePipelineContext(step, result, flowResult.nextStepId, flowResult.pipelineContext), flowResult.nextStepId) + case _ => + val nextStepId = getNextStepId(step, result) + (updatePipelineContext(step, result, nextStepId, ssContext), nextStepId) + } + // run the step finished event + val sfContext = PipelineFlow.handleEvent(newPipelineContext, "pipelineStepFinished", List(pipeline, step, newPipelineContext)) + (nextStepId, sfContext.removeGlobal("stepRetryCount")) + } catch { + case _: Throwable if step.retryLimit.getOrElse(-1) > 0 && stepRetryCount < step.retryLimit.getOrElse(-1) => + // Backoff timer + Thread.sleep(stepRetryCount * 1000) + processStepWithRetry(step, pipeline, ssContext, pipelineContext, stepRetryCount + 1) + case e: Throwable if step.nextStepOnError.isDefined => + // handle exception + val ex = PipelineFlow.handleStepExecutionExceptions(e, pipeline, pipelineContext) + // put exception on the context as the "result" for this step. + val updateContext = updatePipelineContext(step, PipelineStepResponse(Some(ex), None), step.nextStepOnError, ssContext) + (step.nextStepOnError, updateContext) + case e => throw e + } + } + private def processPipelineStep(step: PipelineStep, pipeline: Pipeline, pipelineContext: PipelineContext): Any = { // Create a map of values for each defined parameter val parameterValues: Map[String, Any] = pipelineContext.parameterMapper.createStepParameterMap(step, pipelineContext) diff --git a/metalus-core/src/test/scala/com/acxiom/pipeline/PipelineDefs.scala b/metalus-core/src/test/scala/com/acxiom/pipeline/PipelineDefs.scala index 1e13da70..d149a5a7 100644 --- a/metalus-core/src/test/scala/com/acxiom/pipeline/PipelineDefs.scala +++ b/metalus-core/src/test/scala/com/acxiom/pipeline/PipelineDefs.scala @@ -19,10 +19,22 @@ object PipelineDefs { val DYNAMIC_BRANCH2_STEP: PipelineStep = PipelineStep(Some("DYNAMICBRANCH2STEP"), Some("Global Value Step"), None, Some("Pipeline"), Some(List(Parameter(Some("text"), Some("string"), Some(true), None, Some("!globalInput")))), Some(EngineMeta(Some("MockPipelineSteps.globalVariableStep"))), None, Some("!NON_EXISTENT_VALUE || @DYNAMICBRANCHSTEP")) + val RETRY_STEP: PipelineStep = PipelineStep(Some("RETRYSTEP"), Some("Retry Step"), None, Some("Pipeline"), + Some(List(Parameter(Some("int"), Some("retryCount"), Some(true), None, Some(3)))), + Some(EngineMeta(Some("MockPipelineSteps.retryStep"))), retryLimit = Some(Constants.FOUR)) + val PARROT_STEP: PipelineStep = PipelineStep(Some("PARROTSTEP"), Some("Parrot Step"), None, Some("Pipeline"), + Some(List(Parameter(Some("text"), Some("value"), Some(true), None, Some("error step called!")))), + Some(EngineMeta(Some("MockPipelineSteps.parrotStep")))) val BASIC_PIPELINE = List(TestPipeline(Some("1"), Some("Basic Pipeline"), Some(List(GLOBAL_VALUE_STEP.copy(nextStepId = Some("PAUSESTEP")), PAUSE_STEP)))) + val RETRY_PIPELINE = List(TestPipeline(Some("1"), Some("Retry Pipeline"), + Some(List(RETRY_STEP.copy(nextStepId = Some("RETURNNONESTEP")), RETURN_NOTHING_STEP)))) + + val RETRY_FAILURE_PIPELINE = List(TestPipeline(Some("1"), Some("Retry Failure Pipeline"), + Some(List(RETRY_STEP.copy(nextStepId = Some("RETURNNONESTEP"), nextStepOnError = Some("PARROTSTEP")), RETURN_NOTHING_STEP, PARROT_STEP)))) + val TWO_PIPELINE = List(TestPipeline(Some("0"), Some("First Pipeline"), Some(List(GLOBAL_SINGLE_STEP))), TestPipeline(Some("1"), Some("Second Pipeline"), Some(List(GLOBAL_SINGLE_STEP)))) diff --git a/metalus-core/src/test/scala/com/acxiom/pipeline/SparkSuiteTests.scala b/metalus-core/src/test/scala/com/acxiom/pipeline/SparkSuiteTests.scala index 775ad4a0..e14ef6a3 100644 --- a/metalus-core/src/test/scala/com/acxiom/pipeline/SparkSuiteTests.scala +++ b/metalus-core/src/test/scala/com/acxiom/pipeline/SparkSuiteTests.scala @@ -1,7 +1,5 @@ package com.acxiom.pipeline -import java.io.File - import com.acxiom.pipeline.audits.{AuditType, ExecutionAudit} import com.acxiom.pipeline.drivers.{DefaultPipelineDriver, DriverSetup} import com.acxiom.pipeline.utils.DriverUtils @@ -13,6 +11,8 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.SparkSession import org.scalatest.{BeforeAndAfterAll, FunSpec, GivenWhenThen, Suite} +import java.io.File + class SparkSuiteTests extends FunSpec with BeforeAndAfterAll with GivenWhenThen with Suite { override def beforeAll() { Logger.getLogger("org.apache.spark").setLevel(Level.WARN) @@ -292,6 +292,62 @@ class SparkSuiteTests extends FunSpec with BeforeAndAfterAll with GivenWhenThen } assert(thrown.getMessage.startsWith("Failed to process execution plan after 1 attempts")) } + + it("Should retry a step") { + val args = List("--driverSetupClass", "com.acxiom.pipeline.SparkTestDriverSetup", "--pipeline", "retry") + val results = new ListenerValidations + SparkTestHelper.pipelineListener = new PipelineListener { + override def pipelineStepFinished(pipeline: Pipeline, step: PipelineStep, pipelineContext: PipelineContext): Option[PipelineContext] = { + step.id.getOrElse("") match { + case "RETRYSTEP" => + results.addValidation("RETRYSTEP return value is incorrect", + pipelineContext.parameters.getParametersByPipelineId("1").get.parameters("RETRYSTEP") + .asInstanceOf[PipelineStepResponse].primaryReturn.get.asInstanceOf[String] == "Retried step 3 of 3") + case _ => + } + None + } + + override def registerStepException(exception: PipelineStepException, pipelineContext: PipelineContext): Unit = { + exception match { + case e: Throwable => + results.addValidation("Retry step failed", valid = true) + } + } + } + // Execution should complete without exception + DefaultPipelineDriver.main(args.toArray) + results.validate() + } + + it("Should retry and fail a step") { + val args = List("--driverSetupClass", "com.acxiom.pipeline.SparkTestDriverSetup", "--pipeline", "retryFailure") + val results = new ListenerValidations + SparkTestHelper.pipelineListener = new PipelineListener { + override def pipelineStepFinished(pipeline: Pipeline, step: PipelineStep, pipelineContext: PipelineContext): Option[PipelineContext] = { + step.id.getOrElse("") match { + case "PARROTSTEP" => + results.addValidation("PARROTSTEP return value is incorrect", + pipelineContext.parameters.getParametersByPipelineId("1").get.parameters("PARROTSTEP") + .asInstanceOf[PipelineStepResponse].primaryReturn.get.asInstanceOf[String] == "error step called!") + case "RETURNNONESTEP" => + results.addValidation("RETURNNONESTEP should not have been called", valid = true) + case _ => + } + None + } + + override def registerStepException(exception: PipelineStepException, pipelineContext: PipelineContext): Unit = { + exception match { + case e: Throwable => + results.addValidation("Retry step failed", valid = true) + } + } + } + // Execution should complete without exception + DefaultPipelineDriver.main(args.toArray) + results.validate() + } } describe("PipelineContext") { @@ -365,6 +421,8 @@ case class SparkTestDriverSetup(parameters: Map[String, Any]) extends DriverSetu case "four" => PipelineDefs.FOUR_PIPELINE case "nopause" => PipelineDefs.BASIC_NOPAUSE case "errorTest" => PipelineDefs.ERROR_PIPELINE + case "retry" => PipelineDefs.RETRY_PIPELINE + case "retryFailure" => PipelineDefs.RETRY_FAILURE_PIPELINE case "noPipelines" => List() } @@ -409,4 +467,13 @@ object MockPipelineSteps { throw PipelineException(message = Some("This step should not be called"), pipelineProgress = Some(pipelineContext.getPipelineExecutionInfo)) } + + def retryStep(retryCount: Int, pipelineContext: PipelineContext): String = { + if (pipelineContext.getGlobalAs[Int]("stepRetryCount").getOrElse(-1) == retryCount) { + s"Retried step ${pipelineContext.getGlobalAs[Int]("stepRetryCount").getOrElse(-1)} of $retryCount" + } else { + throw PipelineException(message = Some("Force a retry"), + pipelineProgress = Some(pipelineContext.getPipelineExecutionInfo)) + } + } } From 523fb5973a51d5a2b490593cdb64a3c0f66ef794 Mon Sep 17 00:00:00 2001 From: dafreels Date: Thu, 7 Oct 2021 11:01:45 -0400 Subject: [PATCH 16/24] #254 #258 Implemented a simple retry step to allow multiple steps to be retried. Implemented the copyfile pipeline. --- docs/readme.md | 2 + metalus-common/docs/copyfile.md | 15 ++++ metalus-common/docs/filemanagersteps.md | 26 ++++++- metalus-common/docs/flowutilssteps.md | 17 +++++ metalus-common/readme.md | 2 + .../43bc9450-2689-11ec-9c0c-cbf3549779e5.json | 1 + .../pipeline/steps/FileManagerSteps.scala | 53 +++++++++++-- .../pipeline/steps/FlowUtilsSteps.scala | 30 ++++++++ .../acxiom/pipeline/steps/ApiStepsTests.scala | 9 +-- .../pipeline/steps/FlowUtilsStepsTests.scala | 76 +++++++++++++++++++ 10 files changed, 217 insertions(+), 14 deletions(-) create mode 100644 metalus-common/docs/copyfile.md create mode 100644 metalus-common/docs/flowutilssteps.md create mode 100644 metalus-common/src/main/resources/metadata/pipelines/43bc9450-2689-11ec-9c0c-cbf3549779e5.json create mode 100644 metalus-common/src/main/scala/com/acxiom/pipeline/steps/FlowUtilsSteps.scala create mode 100644 metalus-common/src/test/scala/com/acxiom/pipeline/steps/FlowUtilsStepsTests.scala diff --git a/docs/readme.md b/docs/readme.md index 95a2a441..8b8b4fb0 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -28,6 +28,7 @@ * [DataConnectorSteps](../metalus-common/docs/dataconnectorsteps.md) * [DataFrameSteps](../metalus-common/docs/dataframesteps.md) * [FileManagerSteps](../metalus-common/docs/filemanagersteps.md) + * [FlowUtilsSteps](../metalus-common/docs/flowutilssteps.md) * [HDFSSteps](../metalus-common/docs/hdfssteps.md) * [HiveSteps](../metalus-common/docs/hivesteps.md) * [JavascriptSteps](../metalus-common/docs/javascriptsteps.md) @@ -39,6 +40,7 @@ * [StringSteps](../metalus-common/docs/stringsteps.md) * [TransformationSteps](../metalus-common/docs/transformationsteps.md) * Pipelines/Step Groups + * [Copy File](../metalus-common/docs/copyfile.md) * [SFTP to HDFS](../metalus-common/docs/sftp2hdfs.md) * [Download To Bronze HDFS](../metalus-common/docs/downloadToBronzeHdfs.md) * [DownloadSFTPToHDFSWithDataFrame](../metalus-common/docs/downloadsftptohdfswithdataframe.md) diff --git a/metalus-common/docs/copyfile.md b/metalus-common/docs/copyfile.md new file mode 100644 index 00000000..8e22ab90 --- /dev/null +++ b/metalus-common/docs/copyfile.md @@ -0,0 +1,15 @@ +[Documentation Home](../../docs/readme.md) | [Common Home](../readme.md) + +# Copy File +This pipeline will copy a file from one location to another. + +## General Information +**Id**: _43bc9450-2689-11ec-9c0c-cbf3549779e5_ + +**Name**: _CopyFile_ + +## Required Parameters +* **sourceConnector** - The file connector to use as the source. This is expected to be in the globals. +* **sourceCopyPath** - The path to get the source file. This is expected to be in the globals. +* **destinationConnector** - The file connector to use as the destination. This is expected to be in the globals. +* **destinationCopyPath** - The path to write the destination file. This is expected to be in the globals. diff --git a/metalus-common/docs/filemanagersteps.md b/metalus-common/docs/filemanagersteps.md index b3cab899..99ec5f53 100644 --- a/metalus-common/docs/filemanagersteps.md +++ b/metalus-common/docs/filemanagersteps.md @@ -4,15 +4,37 @@ This object provides steps for working with [FileManager](../../docs/filemanager.md) implementations. ## Copy -This step will copy the contents from the source path using the source FileManager to the +These steps will copy the contents from the source path using the source FileManager to the destination path using the destination FileManager using input and output streams. Full parameter descriptions are listed below: +### Auto Buffering * **srcFS** - The source FileManager used to gain access to the data. * **srcPath** - The path on the source file system containing the data to copy. * **destFS** - The destination FileManger used to write the output data. * **destPath** - The path on the destination file system to write the data. - +### Basic Buffering +Includes the parameters for auto buffering but exposes the input and output buffer sizes: +* **inputBufferSize** - The size of the buffer to use for reading data during copy +* **outputBufferSize** - The size of the buffer to use for writing data during copy +### Advanced Buffering +Includes the parameters for basic buffering but exposes the copy buffer size: +* **copyBufferSize** - The intermediate buffer size to use during copy + +## Compare File Sizes +This step will compare the size of the source and destination files and return -1 if the source file size is smaller +than the destination file size, 0 if they are the same and 1 if the source file size is larger than the source file +size. + +* **srcFS** - The source FileManager. +* **srcPath** - The path on the source file system to the file. +* **destFS** - The destination FileManger. +* **destPath** - The path on the destination file system to the file. +## Delete File +This step will delete a file. + +* **fileManager** - The FileManager. +* **path** - The path to the file being deleted. ## Disconnect File Manager This step provides an easy way to disconnect the FileManager and free any resources. Full parameter descriptions are listed below: diff --git a/metalus-common/docs/flowutilssteps.md b/metalus-common/docs/flowutilssteps.md new file mode 100644 index 00000000..0366ef5d --- /dev/null +++ b/metalus-common/docs/flowutilssteps.md @@ -0,0 +1,17 @@ +[Documentation Home](../../docs/readme.md) | [Common Home](../readme.md) + +# FlowUtilsSteps +These steps offer various tools for manipulating the flow of a [Pipeline](../../docs/pipelines.md). + +## Simple Retry +This step provides a mechanism for determining whether to perform a retry or to stop in the form of a branch step. The +pipeline will need to define a unique name for the counter and the maximum number of retries. The information will be +tracked in the globals. This step is not a replacement for the [step retries](../../docs/pipeline-steps.md#retrylimit) +which provide a mechanism for retrying a single step when an exception occurs. + +### Parameters +* **counterName** - The name of the counter to use for tracking. +* **maxRetries** - The maximum number of retries allowed. +### Results +* **retry** - This branch is used to transition to the step that needs to be retried. +* **stop** - This branch is used to call the step when retries have been exhausted. diff --git a/metalus-common/readme.md b/metalus-common/readme.md index 4324f7cd..ee663adf 100644 --- a/metalus-common/readme.md +++ b/metalus-common/readme.md @@ -11,6 +11,7 @@ using Spark. * [DataFrameSteps](docs/dataframesteps.md) * [DataSteps](docs/datasteps.md) * [FileManagerSteps](docs/filemanagersteps.md) +* [FlowUtilsSteps](docs/flowutilssteps.md) * [HDFSSteps](docs/hdfssteps.md) * [HiveSteps](docs/catalogsteps.md) * [JavascriptSteps](docs/javascriptsteps.md) @@ -23,6 +24,7 @@ using Spark. * [TransformationSteps](docs/transformationsteps.md) ## Pipelines/Step Groups +* [Copy File](docs/copyfile.md) * [SFTP to HDFS](docs/sftp2hdfs.md) * [Download to Bronze HDFS](docs/downloadToBronzeHdfs.md) * [LoadToParquet](docs/loadtoparquet.md) diff --git a/metalus-common/src/main/resources/metadata/pipelines/43bc9450-2689-11ec-9c0c-cbf3549779e5.json b/metalus-common/src/main/resources/metadata/pipelines/43bc9450-2689-11ec-9c0c-cbf3549779e5.json new file mode 100644 index 00000000..27dcdf43 --- /dev/null +++ b/metalus-common/src/main/resources/metadata/pipelines/43bc9450-2689-11ec-9c0c-cbf3549779e5.json @@ -0,0 +1 @@ +{"id":"43bc9450-2689-11ec-9c0c-cbf3549779e5","name":"CopyFile","category":"pipeline","layout":{"GETSOURCE":{"x":493,"y":64},"GETDESTINATION":{"x":493,"y":178},"COPY":{"x":493,"y":292},"VERIFY":{"x":426,"y":406},"TO_STRING":{"x":426,"y":520},"CHECKRESULTS":{"x":426,"y":634},"CLOSESOURCE":{"x":551,"y":886},"CLOSEDESTINATION":{"x":567,"y":1006},"DELETEDESTINATION":{"x":548,"y":698},"RETRY":{"x":713,"y":760}},"steps":[{"id":"GETSOURCE","category":"Connectors","creationDate":"2021-10-06T09:34:28.738Z","description":"Creates a FileManager using the provided FileConnector","displayName":"Create a FileManager","engineMeta":{"spark":"FileManagerSteps.getFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.FileManager"}},"modifiedDate":"2021-10-06T09:34:28.738Z","params":[{"type":"text","name":"fileConnector","required":true,"parameterType":"com.acxiom.pipeline.connectors.FileConnector","description":"The FileConnector to use to create the FileManager implementation","value":"!sourceConnector"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"259a880a-3e12-4843-9f02-2cfc2a05f576","nextStepId":"GETDESTINATION"},{"id":"GETDESTINATION","category":"Connectors","creationDate":"2021-10-06T09:34:28.738Z","description":"Creates a FileManager using the provided FileConnector","displayName":"Create a FileManager","engineMeta":{"spark":"FileManagerSteps.getFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.FileManager"}},"modifiedDate":"2021-10-06T09:34:28.738Z","params":[{"type":"text","name":"fileConnector","required":true,"parameterType":"com.acxiom.pipeline.connectors.FileConnector","description":"The FileConnector to use to create the FileManager implementation","value":"!destinationConnector"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"259a880a-3e12-4843-9f02-2cfc2a05f576","nextStepId":"COPY"},{"id":"COPY","category":"FileManager","creationDate":"2021-10-06T09:34:28.428Z","description":"Copy the contents of the source path to the destination path. This function will call connect on both FileManagers.","displayName":"Copy (auto buffering)","engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}},"modifiedDate":"2021-10-06T10:12:54.804Z","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager","value":"@GETSOURCE"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from","value":"!sourceCopyPath"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager","value":"@GETDESTINATION"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to","value":"!destinationCopyPath"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"0342654c-2722-56fe-ba22-e342169545af","nextStepId":"VERIFY","nextStepOnError":"DELETEDESTINATION"},{"id":"VERIFY","category":"FileManager","creationDate":"2021-10-06T11:05:25.493Z","description":"Compare the file sizes of the source and destination paths","displayName":"Compare File Sizes","engineMeta":{"spark":"FileManagerSteps.compareFileSizes","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Int"}},"modifiedDate":"2021-10-06T11:05:25.493Z","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager","value":"@GETSOURCE"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to the source","value":"!sourceCopyPath"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager","value":"@GETDESTINATION"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to th destination","value":"!destinationCopyPath"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"1af68ab5-a3fe-4afb-b5fa-34e52f7c77f5","nextStepId":"TO_STRING"},{"id":"TO_STRING","category":"String","creationDate":"2021-10-06T09:34:26.365Z","description":"Returns the result of the toString method, can unwrap options","displayName":"To String","engineMeta":{"spark":"StringSteps.toString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"modifiedDate":"2021-10-06T11:05:23.415Z","params":[{"type":"text","name":"value","required":true,"parameterType":"Any","description":"The value to convert","value":"@VERIFY"},{"type":"boolean","name":"unwrapOption","required":false,"parameterType":"Boolean","description":"Boolean indicating whether to unwrap the value from an Option prior to calling toString","value":false}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"b5485d97-d4e8-41a6-8af7-9ce79a435140","nextStepId":"CHECKRESULTS"},{"id":"CHECKRESULTS","category":"Decision","creationDate":"2021-10-06T09:34:26.833Z","description":"Return whether string1 equals string2","displayName":"String Equals","engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"modifiedDate":"2021-10-06T11:05:23.861Z","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to compare","value":"@TO_STRING"},{"type":"text","name":"anotherString","required":true,"parameterType":"String","description":"The other string to compare","value":"0"},{"type":"boolean","name":"caseInsensitive","required":false,"parameterType":"Boolean","description":"Boolean flag to indicate case sensitive compare","value":false},{"type":"result","name":"true","required":false,"value":"CLOSESOURCE"},{"type":"result","name":"false","required":false,"value":"DELETEDESTINATION"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"branch","stepId":"3fabf9ec-5383-4eb3-81af-6092ab7c370d"},{"id":"CLOSESOURCE","category":"FileManager","creationDate":"2021-10-06T09:34:28.672Z","description":"Disconnects a FileManager from the underlying file system","displayName":"Disconnect a FileManager","engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"modifiedDate":"2021-10-06T10:12:55.046Z","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The file manager to disconnect","value":"@GETSOURCE"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"3d1e8519-690c-55f0-bd05-1e7b97fb6633","nextStepId":"CLOSEDESTINATION"},{"id":"CLOSEDESTINATION","category":"FileManager","creationDate":"2021-10-06T09:34:28.672Z","description":"Disconnects a FileManager from the underlying file system","displayName":"Disconnect a FileManager","engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"modifiedDate":"2021-10-06T10:12:55.046Z","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The file manager to disconnect","value":"@GETDESTINATION"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"3d1e8519-690c-55f0-bd05-1e7b97fb6633"},{"id":"DELETEDESTINATION","category":"FileManager","creationDate":"2021-10-06T11:05:25.555Z","description":"Delete a file","displayName":"Delete (file)","engineMeta":{"spark":"FileManagerSteps.deleteFile","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"modifiedDate":"2021-10-06T11:05:25.555Z","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The FileManager","value":"@GETDESTINATION"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the file being deleted","value":"!destinationCopyPath"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"bf2c4df8-a215-480b-87d8-586984e04189","nextStepId":"RETRY"},{"id":"RETRY","category":"RetryLogic","creationDate":"2021-10-07T11:41:30.788Z","description":"Makes a decision to retry or stop based on a named counter","displayName":"Retry (simple)","engineMeta":{"spark":"FlowUtilsSteps.simpleRetry","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"modifiedDate":"2021-10-07T11:41:30.788Z","params":[{"type":"text","name":"counterName","required":true,"parameterType":"String","description":"The name of the counter to use for tracking","value":"COPY_FILE_RETRY"},{"type":"integer","name":"maxRetries","required":true,"parameterType":"Int","description":"The maximum number of retries allowed","value":5},{"type":"result","name":"retry","required":false,"value":"COPY"},{"type":"result","name":"stop","required":false,"value":"CLOSESOURCE"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"branch","stepId":"6ed36f89-35d1-4280-a555-fbcd8dd76bf2"}]} \ No newline at end of file diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/FileManagerSteps.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/FileManagerSteps.scala index 8009948c..a133aa67 100644 --- a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/FileManagerSteps.scala +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/FileManagerSteps.scala @@ -22,10 +22,10 @@ object FileManagerSteps { * @return object with copy results. */ @StepFunction("0342654c-2722-56fe-ba22-e342169545af", - "Copy source contents to destination", + "Copy (auto buffering)", "Copy the contents of the source path to the destination path. This function will call connect on both FileManagers.", "Pipeline", - "InputOutput") + "FileManager") @StepParameters(Map("srcFS" -> StepParameter(None, Some(true), None, None, None, None, Some("The source FileManager")), "srcPath" -> StepParameter(None, Some(true), None, None, None, None, Some("The path to copy from")), "destFS" -> StepParameter(None, Some(true), None, None, None, None, Some("The destination FileManager")), @@ -48,10 +48,10 @@ object FileManagerSteps { * @return object with copy results. */ @StepFunction("c40169a3-1e77-51ab-9e0a-3f24fb98beef", - "Copy source contents to destination with buffering", + "Copy (basic buffering)", "Copy the contents of the source path to the destination path using buffer sizes. This function will call connect on both FileManagers.", "Pipeline", - "InputOutput") + "FileManager") @StepParameters(Map("srcFS" -> StepParameter(None, Some(true), None, None, None, None, Some("The source FileManager")), "srcPath" -> StepParameter(None, Some(true), None, None, None, None, Some("The path to copy from")), "destFS" -> StepParameter(None, Some(true), None, None, None, None, Some("The destination FileManager")), @@ -77,10 +77,10 @@ object FileManagerSteps { * @return object with copy results. */ @StepFunction("f5a24db0-e91b-5c88-8e67-ab5cff09c883", - "Buffered file copy", + "Copy (advanced buffering)", "Copy the contents of the source path to the destination path using full buffer sizes. This function will call connect on both FileManagers.", "Pipeline", - "InputOutput") + "FileManager") @StepParameters(Map("srcFS" -> StepParameter(None, Some(true), None, None, None, None, Some("The source FileManager")), "srcPath" -> StepParameter(None, Some(true), None, None, None, None, Some("The path to copy from")), "destFS" -> StepParameter(None, Some(true), None, None, None, None, Some("The destination FileManager")), @@ -119,6 +119,45 @@ object FileManagerSteps { CopyResults(copied, size, duration, startTime, endTime) } + /** + * Verify that a source path and destination path are the same size. + * + * @param srcFS FileManager for the source file system + * @param srcPath Source path + * @param destFS FileManager for the destination file system + * @param destPath Destination path + * @return true if the source and destination files are the same size + */ + @StepFunction("1af68ab5-a3fe-4afb-b5fa-34e52f7c77f5", + "Compare File Sizes", + "Compare the file sizes of the source and destination paths", + "Pipeline", + "FileManager") + @StepParameters(Map("srcFS" -> StepParameter(None, Some(true), None, None, None, None, Some("The source FileManager")), + "srcPath" -> StepParameter(None, Some(true), None, None, None, None, Some("The path to the source")), + "destFS" -> StepParameter(None, Some(true), None, None, None, None, Some("The destination FileManager")), + "destPath" -> StepParameter(None, Some(true), None, None, None, None, Some("The path to th destination")))) + def compareFileSizes(srcFS: FileManager, srcPath: String, destFS: FileManager, destPath: String): Int = + srcFS.getSize(srcPath).compareTo(destFS.getSize(destPath)) + + /** + * Delete the file using the provided FileManager and Path + * + * @param fileManager The FileManager to use when deleting the file + * @param path The full path to the file + * @return true if the file can be deleted + */ + @StepFunction("bf2c4df8-a215-480b-87d8-586984e04189", + "Delete (file)", + "Delete a file", + "Pipeline", + "FileManager") + @StepParameters(Map("fileManager" -> StepParameter(None, Some(true), None, None, None, None, Some("The FileManager")), + "path" -> StepParameter(None, Some(true), None, None, None, None, Some("The path to the file being deleted")))) + @StepResults(primaryType = "Boolean", secondaryTypes = None) + def deleteFile(fileManager: FileManager, path: String): Boolean = + fileManager.deleteFile(path) + /** * Disconnects a FileManager from the underlying file system * @@ -128,7 +167,7 @@ object FileManagerSteps { "Disconnect a FileManager", "Disconnects a FileManager from the underlying file system", "Pipeline", - "InputOutput") + "FileManager") @StepParameters(Map("fileManager" -> StepParameter(None, Some(true), None, None, None, None, Some("The file manager to disconnect")))) def disconnectFileManager(fileManager: FileManager): Unit = { fileManager.disconnect() diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/FlowUtilsSteps.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/FlowUtilsSteps.scala new file mode 100644 index 00000000..fdd7c567 --- /dev/null +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/FlowUtilsSteps.scala @@ -0,0 +1,30 @@ +package com.acxiom.pipeline.steps + +import com.acxiom.pipeline.annotations._ +import com.acxiom.pipeline.{PipelineContext, PipelineStepResponse} + +@StepObject +object FlowUtilsSteps { + @StepFunction("6ed36f89-35d1-4280-a555-fbcd8dd76bf2", + "Retry (simple)", + "Makes a decision to retry or stop based on a named counter", + "branch", "RetryLogic") + @BranchResults(List("retry", "stop")) + @StepParameters(Map("counterName" -> StepParameter(None, Some(true), None, None, None, None, Some("The name of the counter to use for tracking")), + "maxRetries" -> StepParameter(None, Some(true), None, None, None, None, Some("The maximum number of retries allowed")))) + @StepResults(primaryType = "String", secondaryTypes = Some(Map("$globals.$counterName" -> "Int"))) + def simpleRetry(counterName: String, maxRetries: Int, pipelineContext: PipelineContext): PipelineStepResponse = { + val currentCounter = pipelineContext.getGlobalAs[Int](counterName) + val decision = if (currentCounter.getOrElse(0) < maxRetries) { + "retry" + } else { + "stop" + } + val updateCounter = if (decision == "retry") { + currentCounter.getOrElse(0) + 1 + } else { + currentCounter.getOrElse(0) + } + PipelineStepResponse(Some(decision), Some(Map[String, Any](s"$$globals.$counterName" -> updateCounter))) + } +} diff --git a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/ApiStepsTests.scala b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/ApiStepsTests.scala index f1647405..845a8e35 100644 --- a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/ApiStepsTests.scala +++ b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/ApiStepsTests.scala @@ -1,13 +1,12 @@ package com.acxiom.pipeline.steps -import java.net.{HttpURLConnection, URL} -import java.text.SimpleDateFormat - -import com.acxiom.pipeline.api.{BasicAuthorization, HttpRestClient, SessionAuthorization} +import com.acxiom.pipeline.api.{BasicAuthorization, HttpRestClient} import com.github.tomakehurst.wiremock.WireMockServer -import com.github.tomakehurst.wiremock.client.WireMock.{aMultipart, aResponse, containing, delete, equalTo, get, post, put, urlPathEqualTo} +import com.github.tomakehurst.wiremock.client.WireMock._ import org.scalatest.{BeforeAndAfterAll, FunSpec} +import java.net.HttpURLConnection +import java.text.SimpleDateFormat import scala.io.Source class ApiStepsTests extends FunSpec with BeforeAndAfterAll { diff --git a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/FlowUtilsStepsTests.scala b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/FlowUtilsStepsTests.scala new file mode 100644 index 00000000..90264384 --- /dev/null +++ b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/FlowUtilsStepsTests.scala @@ -0,0 +1,76 @@ +package com.acxiom.pipeline.steps + +import com.acxiom.pipeline._ +import org.apache.commons.io.FileUtils +import org.apache.log4j.{Level, Logger} +import org.apache.spark.SparkConf +import org.apache.spark.sql.SparkSession +import org.scalatest.{BeforeAndAfterAll, FunSpec} + +import java.nio.file.{Files, Path} + +class FlowUtilsStepsTests extends FunSpec with BeforeAndAfterAll { + val MASTER = "local[2]" + val APPNAME = "flow-utils-steps-spark" + var sparkConf: SparkConf = _ + var sparkSession: SparkSession = _ + var pipelineContext: PipelineContext = _ + val sparkLocalDir: Path = Files.createTempDirectory("sparkLocal") + + val STRING_STEP: PipelineStep = PipelineStep(Some("STRINGSTEP"), Some("String Step"), None, Some("Pipeline"), + Some(List(Parameter(Some("text"), Some("value"), Some(true), None, Some("lowercase")))), + Some(EngineMeta(Some("StringSteps.toUpperCase"))), Some("RETRY")) + + val RETRY_STEP: PipelineStep = PipelineStep(Some("RETRY"), Some("Retry Step"), None, Some("branch"), + Some(List(Parameter(Some("text"), Some("counterName"), Some(true), None, Some("TEST_RETRY_COUNTER")), + Parameter(Some("int"), Some("maxRetries"), Some(true), None, Some(Constants.FIVE)), + Parameter(Some("result"), Some("retry"), Some(true), None, Some("STRINGSTEP")))), + Some(EngineMeta(Some("FlowUtilsSteps.simpleRetry"))), None) + + override def beforeAll(): Unit = { + Logger.getLogger("org.apache.spark").setLevel(Level.WARN) + Logger.getLogger("org.apache.hadoop").setLevel(Level.WARN) + Logger.getLogger("com.acxiom.pipeline").setLevel(Level.DEBUG) + + sparkConf = new SparkConf() + .setMaster(MASTER) + .setAppName(APPNAME) + .set("spark.local.dir", sparkLocalDir.toFile.getAbsolutePath) + sparkSession = SparkSession.builder().config(sparkConf).getOrCreate() + + pipelineContext = PipelineContext(Some(sparkConf), Some(sparkSession), Some(Map[String, Any]()), + PipelineSecurityManager(), + PipelineParameters(List(PipelineParameter("0", Map[String, Any]()), PipelineParameter("1", Map[String, Any]()))), + Some(List("com.acxiom.pipeline.steps")), + PipelineStepMapper(), + Some(DefaultPipelineListener()), + Some(sparkSession.sparkContext.collectionAccumulator[PipelineStepMessage]("stepMessages"))) + } + + override def afterAll(): Unit = { + sparkSession.sparkContext.cancelAllJobs() + sparkSession.sparkContext.stop() + sparkSession.stop() + + Logger.getRootLogger.setLevel(Level.INFO) + // cleanup spark directories + FileUtils.deleteDirectory(sparkLocalDir.toFile) + } + + describe("FlowUtilsSteps - Retry") { + it("Should retry and trigger stop") { + val pipeline = Pipeline(Some("testPipeline"), Some("retryPipeline"), Some(List(STRING_STEP, RETRY_STEP))) + val initialPipelineContext = PipelineContext(Some(sparkConf), Some(sparkSession), Some(Map[String, Any]()), + PipelineSecurityManager(), + PipelineParameters(List(PipelineParameter("0", Map[String, Any]()), PipelineParameter("1", Map[String, Any]()))), + Some(List("com.acxiom.pipeline.steps")), + PipelineStepMapper(), + Some(DefaultPipelineListener()), + Some(sparkSession.sparkContext.collectionAccumulator[PipelineStepMessage]("stepMessages"))) + val result = PipelineExecutor.executePipelines(List(pipeline), None, initialPipelineContext) + val counter = result.pipelineContext.getGlobalAs[Int]("TEST_RETRY_COUNTER") + assert(counter.isDefined) + assert(counter.get == Constants.FIVE) + } + } +} From 3abcdc30e746c70693dc57001456c448262a7323 Mon Sep 17 00:00:00 2001 From: dafreels Date: Thu, 7 Oct 2021 11:25:15 -0400 Subject: [PATCH 17/24] #258 Updated copy pipeline copy step to make 5 attempts when failed and updated documentation. --- metalus-common/docs/copyfile.md | 3 +- .../43bc9450-2689-11ec-9c0c-cbf3549779e5.json | 465 +++++++++++++++++- 2 files changed, 466 insertions(+), 2 deletions(-) diff --git a/metalus-common/docs/copyfile.md b/metalus-common/docs/copyfile.md index 8e22ab90..c1da8726 100644 --- a/metalus-common/docs/copyfile.md +++ b/metalus-common/docs/copyfile.md @@ -1,7 +1,8 @@ [Documentation Home](../../docs/readme.md) | [Common Home](../readme.md) # Copy File -This pipeline will copy a file from one location to another. +This pipeline will copy a file from one location to another. The download will make up to 5 attempts when a failure +occurs. A verification check will ensure the downloaded file is the same size and if not, will re-download up to 5 times. ## General Information **Id**: _43bc9450-2689-11ec-9c0c-cbf3549779e5_ diff --git a/metalus-common/src/main/resources/metadata/pipelines/43bc9450-2689-11ec-9c0c-cbf3549779e5.json b/metalus-common/src/main/resources/metadata/pipelines/43bc9450-2689-11ec-9c0c-cbf3549779e5.json index 27dcdf43..6e424dd0 100644 --- a/metalus-common/src/main/resources/metadata/pipelines/43bc9450-2689-11ec-9c0c-cbf3549779e5.json +++ b/metalus-common/src/main/resources/metadata/pipelines/43bc9450-2689-11ec-9c0c-cbf3549779e5.json @@ -1 +1,464 @@ -{"id":"43bc9450-2689-11ec-9c0c-cbf3549779e5","name":"CopyFile","category":"pipeline","layout":{"GETSOURCE":{"x":493,"y":64},"GETDESTINATION":{"x":493,"y":178},"COPY":{"x":493,"y":292},"VERIFY":{"x":426,"y":406},"TO_STRING":{"x":426,"y":520},"CHECKRESULTS":{"x":426,"y":634},"CLOSESOURCE":{"x":551,"y":886},"CLOSEDESTINATION":{"x":567,"y":1006},"DELETEDESTINATION":{"x":548,"y":698},"RETRY":{"x":713,"y":760}},"steps":[{"id":"GETSOURCE","category":"Connectors","creationDate":"2021-10-06T09:34:28.738Z","description":"Creates a FileManager using the provided FileConnector","displayName":"Create a FileManager","engineMeta":{"spark":"FileManagerSteps.getFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.FileManager"}},"modifiedDate":"2021-10-06T09:34:28.738Z","params":[{"type":"text","name":"fileConnector","required":true,"parameterType":"com.acxiom.pipeline.connectors.FileConnector","description":"The FileConnector to use to create the FileManager implementation","value":"!sourceConnector"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"259a880a-3e12-4843-9f02-2cfc2a05f576","nextStepId":"GETDESTINATION"},{"id":"GETDESTINATION","category":"Connectors","creationDate":"2021-10-06T09:34:28.738Z","description":"Creates a FileManager using the provided FileConnector","displayName":"Create a FileManager","engineMeta":{"spark":"FileManagerSteps.getFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.FileManager"}},"modifiedDate":"2021-10-06T09:34:28.738Z","params":[{"type":"text","name":"fileConnector","required":true,"parameterType":"com.acxiom.pipeline.connectors.FileConnector","description":"The FileConnector to use to create the FileManager implementation","value":"!destinationConnector"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"259a880a-3e12-4843-9f02-2cfc2a05f576","nextStepId":"COPY"},{"id":"COPY","category":"FileManager","creationDate":"2021-10-06T09:34:28.428Z","description":"Copy the contents of the source path to the destination path. This function will call connect on both FileManagers.","displayName":"Copy (auto buffering)","engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}},"modifiedDate":"2021-10-06T10:12:54.804Z","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager","value":"@GETSOURCE"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from","value":"!sourceCopyPath"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager","value":"@GETDESTINATION"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to","value":"!destinationCopyPath"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"0342654c-2722-56fe-ba22-e342169545af","nextStepId":"VERIFY","nextStepOnError":"DELETEDESTINATION"},{"id":"VERIFY","category":"FileManager","creationDate":"2021-10-06T11:05:25.493Z","description":"Compare the file sizes of the source and destination paths","displayName":"Compare File Sizes","engineMeta":{"spark":"FileManagerSteps.compareFileSizes","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Int"}},"modifiedDate":"2021-10-06T11:05:25.493Z","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager","value":"@GETSOURCE"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to the source","value":"!sourceCopyPath"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager","value":"@GETDESTINATION"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to th destination","value":"!destinationCopyPath"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"1af68ab5-a3fe-4afb-b5fa-34e52f7c77f5","nextStepId":"TO_STRING"},{"id":"TO_STRING","category":"String","creationDate":"2021-10-06T09:34:26.365Z","description":"Returns the result of the toString method, can unwrap options","displayName":"To String","engineMeta":{"spark":"StringSteps.toString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"modifiedDate":"2021-10-06T11:05:23.415Z","params":[{"type":"text","name":"value","required":true,"parameterType":"Any","description":"The value to convert","value":"@VERIFY"},{"type":"boolean","name":"unwrapOption","required":false,"parameterType":"Boolean","description":"Boolean indicating whether to unwrap the value from an Option prior to calling toString","value":false}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"b5485d97-d4e8-41a6-8af7-9ce79a435140","nextStepId":"CHECKRESULTS"},{"id":"CHECKRESULTS","category":"Decision","creationDate":"2021-10-06T09:34:26.833Z","description":"Return whether string1 equals string2","displayName":"String Equals","engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"modifiedDate":"2021-10-06T11:05:23.861Z","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to compare","value":"@TO_STRING"},{"type":"text","name":"anotherString","required":true,"parameterType":"String","description":"The other string to compare","value":"0"},{"type":"boolean","name":"caseInsensitive","required":false,"parameterType":"Boolean","description":"Boolean flag to indicate case sensitive compare","value":false},{"type":"result","name":"true","required":false,"value":"CLOSESOURCE"},{"type":"result","name":"false","required":false,"value":"DELETEDESTINATION"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"branch","stepId":"3fabf9ec-5383-4eb3-81af-6092ab7c370d"},{"id":"CLOSESOURCE","category":"FileManager","creationDate":"2021-10-06T09:34:28.672Z","description":"Disconnects a FileManager from the underlying file system","displayName":"Disconnect a FileManager","engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"modifiedDate":"2021-10-06T10:12:55.046Z","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The file manager to disconnect","value":"@GETSOURCE"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"3d1e8519-690c-55f0-bd05-1e7b97fb6633","nextStepId":"CLOSEDESTINATION"},{"id":"CLOSEDESTINATION","category":"FileManager","creationDate":"2021-10-06T09:34:28.672Z","description":"Disconnects a FileManager from the underlying file system","displayName":"Disconnect a FileManager","engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"modifiedDate":"2021-10-06T10:12:55.046Z","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The file manager to disconnect","value":"@GETDESTINATION"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"3d1e8519-690c-55f0-bd05-1e7b97fb6633"},{"id":"DELETEDESTINATION","category":"FileManager","creationDate":"2021-10-06T11:05:25.555Z","description":"Delete a file","displayName":"Delete (file)","engineMeta":{"spark":"FileManagerSteps.deleteFile","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"modifiedDate":"2021-10-06T11:05:25.555Z","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The FileManager","value":"@GETDESTINATION"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the file being deleted","value":"!destinationCopyPath"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"Pipeline","stepId":"bf2c4df8-a215-480b-87d8-586984e04189","nextStepId":"RETRY"},{"id":"RETRY","category":"RetryLogic","creationDate":"2021-10-07T11:41:30.788Z","description":"Makes a decision to retry or stop based on a named counter","displayName":"Retry (simple)","engineMeta":{"spark":"FlowUtilsSteps.simpleRetry","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"modifiedDate":"2021-10-07T11:41:30.788Z","params":[{"type":"text","name":"counterName","required":true,"parameterType":"String","description":"The name of the counter to use for tracking","value":"COPY_FILE_RETRY"},{"type":"integer","name":"maxRetries","required":true,"parameterType":"Int","description":"The maximum number of retries allowed","value":5},{"type":"result","name":"retry","required":false,"value":"COPY"},{"type":"result","name":"stop","required":false,"value":"CLOSESOURCE"}],"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"type":"branch","stepId":"6ed36f89-35d1-4280-a555-fbcd8dd76bf2"}]} \ No newline at end of file +{ + "id": "43bc9450-2689-11ec-9c0c-cbf3549779e5", + "name": "CopyFile", + "category": "pipeline", + "layout": { + "GETSOURCE": { + "x": 493, + "y": 64 + }, + "GETDESTINATION": { + "x": 493, + "y": 178 + }, + "COPY": { + "x": 493, + "y": 292 + }, + "VERIFY": { + "x": 426, + "y": 406 + }, + "TO_STRING": { + "x": 426, + "y": 520 + }, + "CHECKRESULTS": { + "x": 426, + "y": 634 + }, + "CLOSESOURCE": { + "x": 551, + "y": 886 + }, + "CLOSEDESTINATION": { + "x": 567, + "y": 1006 + }, + "DELETEDESTINATION": { + "x": 548, + "y": 698 + }, + "RETRY": { + "x": 713, + "y": 760 + } + }, + "steps": [ + { + "id": "GETSOURCE", + "category": "Connectors", + "creationDate": "2021-10-06T09:34:28.738Z", + "description": "Creates a FileManager using the provided FileConnector", + "displayName": "Create a FileManager", + "engineMeta": { + "spark": "FileManagerSteps.getFileManager", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "com.acxiom.pipeline.fs.FileManager" + } + }, + "modifiedDate": "2021-10-06T09:34:28.738Z", + "params": [ + { + "type": "text", + "name": "fileConnector", + "required": true, + "parameterType": "com.acxiom.pipeline.connectors.FileConnector", + "description": "The FileConnector to use to create the FileManager implementation", + "value": "!sourceConnector" + } + ], + "tags": [ + "metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar" + ], + "type": "Pipeline", + "stepId": "259a880a-3e12-4843-9f02-2cfc2a05f576", + "nextStepId": "GETDESTINATION" + }, + { + "id": "GETDESTINATION", + "category": "Connectors", + "creationDate": "2021-10-06T09:34:28.738Z", + "description": "Creates a FileManager using the provided FileConnector", + "displayName": "Create a FileManager", + "engineMeta": { + "spark": "FileManagerSteps.getFileManager", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "com.acxiom.pipeline.fs.FileManager" + } + }, + "modifiedDate": "2021-10-06T09:34:28.738Z", + "params": [ + { + "type": "text", + "name": "fileConnector", + "required": true, + "parameterType": "com.acxiom.pipeline.connectors.FileConnector", + "description": "The FileConnector to use to create the FileManager implementation", + "value": "!destinationConnector" + } + ], + "tags": [ + "metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar" + ], + "type": "Pipeline", + "stepId": "259a880a-3e12-4843-9f02-2cfc2a05f576", + "nextStepId": "COPY" + }, + { + "id": "COPY", + "category": "FileManager", + "creationDate": "2021-10-06T09:34:28.428Z", + "description": "Copy the contents of the source path to the destination path. This function will call connect on both FileManagers.", + "displayName": "Copy (auto buffering)", + "retryLimit": 5, + "engineMeta": { + "spark": "FileManagerSteps.copy", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "com.acxiom.pipeline.steps.CopyResults" + } + }, + "modifiedDate": "2021-10-06T10:12:54.804Z", + "params": [ + { + "type": "text", + "name": "srcFS", + "required": true, + "parameterType": "com.acxiom.pipeline.fs.FileManager", + "description": "The source FileManager", + "value": "@GETSOURCE" + }, + { + "type": "text", + "name": "srcPath", + "required": true, + "parameterType": "String", + "description": "The path to copy from", + "value": "!sourceCopyPath" + }, + { + "type": "text", + "name": "destFS", + "required": true, + "parameterType": "com.acxiom.pipeline.fs.FileManager", + "description": "The destination FileManager", + "value": "@GETDESTINATION" + }, + { + "type": "text", + "name": "destPath", + "required": true, + "parameterType": "String", + "description": "The path to copy to", + "value": "!destinationCopyPath" + } + ], + "tags": [ + "metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar" + ], + "type": "Pipeline", + "stepId": "0342654c-2722-56fe-ba22-e342169545af", + "nextStepId": "VERIFY", + "nextStepOnError": "DELETEDESTINATION" + }, + { + "id": "VERIFY", + "category": "FileManager", + "creationDate": "2021-10-06T11:05:25.493Z", + "description": "Compare the file sizes of the source and destination paths", + "displayName": "Compare File Sizes", + "engineMeta": { + "spark": "FileManagerSteps.compareFileSizes", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "Int" + } + }, + "modifiedDate": "2021-10-06T11:05:25.493Z", + "params": [ + { + "type": "text", + "name": "srcFS", + "required": true, + "parameterType": "com.acxiom.pipeline.fs.FileManager", + "description": "The source FileManager", + "value": "@GETSOURCE" + }, + { + "type": "text", + "name": "srcPath", + "required": true, + "parameterType": "String", + "description": "The path to the source", + "value": "!sourceCopyPath" + }, + { + "type": "text", + "name": "destFS", + "required": true, + "parameterType": "com.acxiom.pipeline.fs.FileManager", + "description": "The destination FileManager", + "value": "@GETDESTINATION" + }, + { + "type": "text", + "name": "destPath", + "required": true, + "parameterType": "String", + "description": "The path to th destination", + "value": "!destinationCopyPath" + } + ], + "tags": [ + "metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar" + ], + "type": "Pipeline", + "stepId": "1af68ab5-a3fe-4afb-b5fa-34e52f7c77f5", + "nextStepId": "TO_STRING" + }, + { + "id": "TO_STRING", + "category": "String", + "creationDate": "2021-10-06T09:34:26.365Z", + "description": "Returns the result of the toString method, can unwrap options", + "displayName": "To String", + "engineMeta": { + "spark": "StringSteps.toString", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "String" + } + }, + "modifiedDate": "2021-10-06T11:05:23.415Z", + "params": [ + { + "type": "text", + "name": "value", + "required": true, + "parameterType": "Any", + "description": "The value to convert", + "value": "@VERIFY" + }, + { + "type": "boolean", + "name": "unwrapOption", + "required": false, + "parameterType": "Boolean", + "description": "Boolean indicating whether to unwrap the value from an Option prior to calling toString", + "value": false + } + ], + "tags": [ + "metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar" + ], + "type": "Pipeline", + "stepId": "b5485d97-d4e8-41a6-8af7-9ce79a435140", + "nextStepId": "CHECKRESULTS" + }, + { + "id": "CHECKRESULTS", + "category": "Decision", + "creationDate": "2021-10-06T09:34:26.833Z", + "description": "Return whether string1 equals string2", + "displayName": "String Equals", + "engineMeta": { + "spark": "StringSteps.stringEquals", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "Boolean" + } + }, + "modifiedDate": "2021-10-06T11:05:23.861Z", + "params": [ + { + "type": "text", + "name": "string", + "required": true, + "parameterType": "String", + "description": "The string to compare", + "value": "@TO_STRING" + }, + { + "type": "text", + "name": "anotherString", + "required": true, + "parameterType": "String", + "description": "The other string to compare", + "value": "0" + }, + { + "type": "boolean", + "name": "caseInsensitive", + "required": false, + "parameterType": "Boolean", + "description": "Boolean flag to indicate case sensitive compare", + "value": false + }, + { + "type": "result", + "name": "true", + "required": false, + "value": "CLOSESOURCE" + }, + { + "type": "result", + "name": "false", + "required": false, + "value": "DELETEDESTINATION" + } + ], + "tags": [ + "metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar" + ], + "type": "branch", + "stepId": "3fabf9ec-5383-4eb3-81af-6092ab7c370d" + }, + { + "id": "CLOSESOURCE", + "category": "FileManager", + "creationDate": "2021-10-06T09:34:28.672Z", + "description": "Disconnects a FileManager from the underlying file system", + "displayName": "Disconnect a FileManager", + "engineMeta": { + "spark": "FileManagerSteps.disconnectFileManager", + "pkg": "com.acxiom.pipeline.steps" + }, + "modifiedDate": "2021-10-06T10:12:55.046Z", + "params": [ + { + "type": "text", + "name": "fileManager", + "required": true, + "parameterType": "com.acxiom.pipeline.fs.FileManager", + "description": "The file manager to disconnect", + "value": "@GETSOURCE" + } + ], + "tags": [ + "metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar" + ], + "type": "Pipeline", + "stepId": "3d1e8519-690c-55f0-bd05-1e7b97fb6633", + "nextStepId": "CLOSEDESTINATION" + }, + { + "id": "CLOSEDESTINATION", + "category": "FileManager", + "creationDate": "2021-10-06T09:34:28.672Z", + "description": "Disconnects a FileManager from the underlying file system", + "displayName": "Disconnect a FileManager", + "engineMeta": { + "spark": "FileManagerSteps.disconnectFileManager", + "pkg": "com.acxiom.pipeline.steps" + }, + "modifiedDate": "2021-10-06T10:12:55.046Z", + "params": [ + { + "type": "text", + "name": "fileManager", + "required": true, + "parameterType": "com.acxiom.pipeline.fs.FileManager", + "description": "The file manager to disconnect", + "value": "@GETDESTINATION" + } + ], + "tags": [ + "metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar" + ], + "type": "Pipeline", + "stepId": "3d1e8519-690c-55f0-bd05-1e7b97fb6633" + }, + { + "id": "DELETEDESTINATION", + "category": "FileManager", + "creationDate": "2021-10-06T11:05:25.555Z", + "description": "Delete a file", + "displayName": "Delete (file)", + "engineMeta": { + "spark": "FileManagerSteps.deleteFile", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "Boolean" + } + }, + "modifiedDate": "2021-10-06T11:05:25.555Z", + "params": [ + { + "type": "text", + "name": "fileManager", + "required": true, + "parameterType": "com.acxiom.pipeline.fs.FileManager", + "description": "The FileManager", + "value": "@GETDESTINATION" + }, + { + "type": "text", + "name": "path", + "required": true, + "parameterType": "String", + "description": "The path to the file being deleted", + "value": "!destinationCopyPath" + } + ], + "tags": [ + "metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar" + ], + "type": "Pipeline", + "stepId": "bf2c4df8-a215-480b-87d8-586984e04189", + "nextStepId": "RETRY" + }, + { + "id": "RETRY", + "category": "RetryLogic", + "creationDate": "2021-10-07T11:41:30.788Z", + "description": "Makes a decision to retry or stop based on a named counter", + "displayName": "Retry (simple)", + "engineMeta": { + "spark": "FlowUtilsSteps.simpleRetry", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "com.acxiom.pipeline.PipelineStepResponse" + } + }, + "modifiedDate": "2021-10-07T11:41:30.788Z", + "params": [ + { + "type": "text", + "name": "counterName", + "required": true, + "parameterType": "String", + "description": "The name of the counter to use for tracking", + "value": "COPY_FILE_RETRY" + }, + { + "type": "integer", + "name": "maxRetries", + "required": true, + "parameterType": "Int", + "description": "The maximum number of retries allowed", + "value": 5 + }, + { + "type": "result", + "name": "retry", + "required": false, + "value": "COPY" + }, + { + "type": "result", + "name": "stop", + "required": false, + "value": "CLOSESOURCE" + } + ], + "tags": [ + "metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar" + ], + "type": "branch", + "stepId": "6ed36f89-35d1-4280-a555-fbcd8dd76bf2" + } + ] +} From f6ba26f24c30a074b13377680538b649143cc7e6 Mon Sep 17 00:00:00 2001 From: dafreels Date: Fri, 8 Oct 2021 08:29:04 -0400 Subject: [PATCH 18/24] #252 Created a load to bronze pipeline that uses connectors for reusability. --- docs/dataconnectors.md | 49 +-- docs/readme.md | 3 +- metalus-common/docs/dataconnectorsteps.md | 4 +- metalus-common/docs/dataoptions.md | 26 ++ metalus-common/docs/loadtobronze.md | 23 ++ .../a9f62840-2827-11ec-9c0c-cbf3549779e5.json | 381 ++++++++++++++++++ 6 files changed, 441 insertions(+), 45 deletions(-) create mode 100644 metalus-common/docs/dataoptions.md create mode 100644 metalus-common/docs/loadtobronze.md create mode 100644 metalus-common/src/main/resources/metadata/pipelines/a9f62840-2827-11ec-9c0c-cbf3549779e5.json diff --git a/docs/dataconnectors.md b/docs/dataconnectors.md index 95577388..e26628b4 100644 --- a/docs/dataconnectors.md +++ b/docs/dataconnectors.md @@ -22,9 +22,7 @@ instead relying on the permissions of the cluster. Below is an example setup: #### Scala ```scala -val connector = HDFSDataConnector("my-connector", None, None, - DataFrameReaderOptions(format = "csv"), - DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) +val connector = HDFSDataConnector("my-connector", None, None) ``` #### Globals JSON ```json @@ -32,16 +30,7 @@ val connector = HDFSDataConnector("my-connector", None, None, "myConnector": { "className": "com.acxiom.pipeline.connectors.HDFSDataConnector", "object": { - "name": "my-connector", - "readOptions": { - "format": "csv" - }, - "writeOptions": { - "format": "csv", - "options": { - "delimiter": "þ" - } - } + "name": "my-connector" } } } @@ -50,9 +39,7 @@ val connector = HDFSDataConnector("my-connector", None, None, This connector provides access to S3. Below is an example setup that expects a secrets manager credential provider: #### Scala ```scala -val connector = S3DataConnector("my-connector", Some("my-credential-name-for-secrets-manager"), None, - DataFrameReaderOptions(format = "csv"), - DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) +val connector = S3DataConnector("my-connector", Some("my-credential-name-for-secrets-manager"), None) ``` #### Globals JSON ```json @@ -61,16 +48,7 @@ val connector = S3DataConnector("my-connector", Some("my-credential-name-for-sec "className": "com.acxiom.aws.pipeline.connectors.S3DataConnector", "object": { "name": "my-connector", - "credentialName": "my-credential-name-for-secrets-manager", - "readOptions": { - "format": "csv" - }, - "writeOptions": { - "format": "csv", - "options": { - "delimiter": "þ" - } - } + "credentialName": "my-credential-name-for-secrets-manager" } } } @@ -79,9 +57,7 @@ val connector = S3DataConnector("my-connector", Some("my-credential-name-for-sec This connector provides access to GCS. Below is an example setup that expects a secrets manager credential provider: #### Scala ```scala -val connector = GCSDataConnector("my-connector", Some("my-credential-name-for-secrets-manager"), None, - DataFrameReaderOptions(format = "csv"), - DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) +val connector = GCSDataConnector("my-connector", Some("my-credential-name-for-secrets-manager"), None) ``` #### Globals JSON ```json @@ -90,16 +66,7 @@ val connector = GCSDataConnector("my-connector", Some("my-credential-name-for-se "className": "com.acxiom.gcp.pipeline.connectors.GCSDataConnector", "object": { "name": "my-connector", - "credentialName": "my-credential-name-for-secrets-manager", - "readOptions": { - "format": "csv" - }, - "writeOptions": { - "format": "csv", - "options": { - "delimiter": "þ" - } - } + "credentialName": "my-credential-name-for-secrets-manager" } } } @@ -108,9 +75,7 @@ val connector = GCSDataConnector("my-connector", Some("my-credential-name-for-se This connector provides access to BigQuery. Below is an example setup that expects a secrets manager credential provider: #### Scala ```scala -val connector = BigQueryDataConnector("temp-bucket-name", "my-connector", Some("my-credential-name-for-secrets-manager"), None, - DataFrameReaderOptions(format = "csv"), - DataFrameWriterOptions(format = "csv", options = Some(Map("delimiter" -> "þ")))) +val connector = BigQueryDataConnector("temp-bucket-name", "my-connector", Some("my-credential-name-for-secrets-manager"), None) ``` #### Globals JSON ```json diff --git a/docs/readme.md b/docs/readme.md index 8b8b4fb0..f8443591 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -40,7 +40,8 @@ * [StringSteps](../metalus-common/docs/stringsteps.md) * [TransformationSteps](../metalus-common/docs/transformationsteps.md) * Pipelines/Step Groups - * [Copy File](../metalus-common/docs/copyfile.md) + * [Copy File](../metalus-common/docs/copyfile.md) * _Uses new connectors api_ + * [Load to Bronze](../metalus-common/docs/loadtobronze.md) * _Uses new connectors api_ * [SFTP to HDFS](../metalus-common/docs/sftp2hdfs.md) * [Download To Bronze HDFS](../metalus-common/docs/downloadToBronzeHdfs.md) * [DownloadSFTPToHDFSWithDataFrame](../metalus-common/docs/downloadsftptohdfswithdataframe.md) diff --git a/metalus-common/docs/dataconnectorsteps.md b/metalus-common/docs/dataconnectorsteps.md index cdacb542..043fd5cf 100644 --- a/metalus-common/docs/dataconnectorsteps.md +++ b/metalus-common/docs/dataconnectorsteps.md @@ -9,11 +9,11 @@ This function will write a given DataFrame using the provided DataConnector. Ful * **dataFrame** - A dataFrame to be written. * **connector** - The DataConnector to use when writing data. * **destination** - An optional destination. -* **options** - Optional DataFrameWriterOptions object to configure the DataFrameWriter +* **options** - Optional [DataFrameWriterOptions](dataoptions.md#dataframe-writer-options) object to configure the DataFrameWriter ## Load DataFrame This function will load a DataFrame using the provided DataConnector. Full parameter descriptions are listed below: * **source** - An optional source. * **connector** - The DataConnector to use when loading data. -* **options** - Optional DataFrameReaderOptions object to configure the DataFrameReader +* **options** - Optional [DataFrameReaderOptions](dataoptions.md#dataframe-reader-options) object to configure the DataFrameReader diff --git a/metalus-common/docs/dataoptions.md b/metalus-common/docs/dataoptions.md new file mode 100644 index 00000000..31f34103 --- /dev/null +++ b/metalus-common/docs/dataoptions.md @@ -0,0 +1,26 @@ +[Documentation Home](../../docs/readme.md) | [Common Home](../readme.md) + +# Data Options +Read and write options need to be easy to configure and reusable across steps and pipelines. The DataFrameReaderOptions +and DataFrameWriterOptions are provided as a way to easily model reusable instructions. + +## DataFrame Reader Options +This object represents options that will be translated into Spark settings at read time. + +* *format* - The file format to use. Defaulted to _parquet_. +* *schema* - An optional _com.acxiom.pipeline.steps.Schema_ which will be applied to the DataFrame. +* *options* - Optional map of settings to pass to the underlying Spark data source. +## DataFrame Writer Options +This object represents options that will be translated into Spark settings at write time. + +* *format* - The file format to use. Defaulted to _parquet_. +* *saveMode* - The mode when writing a DataFrame. Defaulted to "Overwrite" +* *options* - Optional map of settings to pass to the underlying Spark data source. +* *bucketingOptions* - Optional BucketingOptions object for configuring Bucketing +* *partitionBy* - Optional list of columns for partitioning. +* *sortBy* - Optional list of columns for sorting. +## Bucketing Options +This object represents the options used to bucket data when it is being written by Spark. + +* *numBuckets* - The number of buckets (partition) to build. +* *columns* - A list of columns to bucket by. diff --git a/metalus-common/docs/loadtobronze.md b/metalus-common/docs/loadtobronze.md new file mode 100644 index 00000000..7d49c73e --- /dev/null +++ b/metalus-common/docs/loadtobronze.md @@ -0,0 +1,23 @@ +[Documentation Home](../../docs/readme.md) | [Common Home](../readme.md) + +# Load to Bronze +This pipeline will load source data to the _bronze_ zone. During the load the pipeline will optionally standardize +column names, add a unique record id and a static file id column. + +## General Information +**Id**: _a9f62840-2827-11ec-9c0c-cbf3549779e5_ + +**Name**: _LoadToBronze_ + +## Required Parameters (all parameters should be part of the globals) +* **sourceBronzeConnector** - The data connector to use as the source. +* **sourceBronzePath** - The path to get the source data. +* **sourceBronzeReadOptions** - The [options](dataoptions.md#dataframe-reader-options) that describe the source data and any settings to use during the read. +* **executeColumnCleanup** - Optional boolean indicating that column names should be standardized. _Defaults to true._ +* **addRecordId** - Optional boolean indicating that a unique record id should be added. The uniqueness is only within + the current source data and does not consider the destination data. _Defaults to true._ +* **addFileId** - Optional boolean indicating that a static column containing the provided _fileId_ should be added to + each record. _Defaults to true._ +* **destinationBronzeConnector** - The data connector to use as the destination. +* **destinationBronzePath** - The path to write the data. +* **destinationBronzeWriteOptions** - The [options](dataoptions.md#dataframe-writer-options) to use during the write. diff --git a/metalus-common/src/main/resources/metadata/pipelines/a9f62840-2827-11ec-9c0c-cbf3549779e5.json b/metalus-common/src/main/resources/metadata/pipelines/a9f62840-2827-11ec-9c0c-cbf3549779e5.json new file mode 100644 index 00000000..56d4a3c6 --- /dev/null +++ b/metalus-common/src/main/resources/metadata/pipelines/a9f62840-2827-11ec-9c0c-cbf3549779e5.json @@ -0,0 +1,381 @@ +{ + "id": "a9f62840-2827-11ec-9c0c-cbf3549779e5", + "name": "LoadToBronze", + "category": "pipeline", + "layout": { + "Load": { + "x": 493, + "y": 64 + }, + "ExecuteColumnCleanup": { + "x": 493, + "y": 178 + }, + "StandardizeColumnNames": { + "x": 526.5, + "y": 292 + }, + "AddRecordIdDecision": { + "x": 493, + "y": 406 + }, + "AddRecordId": { + "x": 526.5, + "y": 520 + }, + "AddFileIdDecision": { + "x": 493, + "y": 634 + }, + "AddFileId": { + "x": 526.5, + "y": 748 + }, + "Write": { + "x": 493, + "y": 862 + } + }, + "steps": [ + { + "id": "Load", + "category": "Connectors", + "creationDate": "2021-10-06T09:34:22.488Z", + "description": "This step will create a DataFrame using the given DataConnector", + "displayName": "Load", + "engineMeta": { + "spark": "DataConnectorSteps.loadDataFrame", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "org.apache.spark.sql.DataFrame" + } + }, + "modifiedDate": "2021-10-07T11:41:28.885Z", + "params": [ + { + "type": "text", + "name": "connector", + "required": true, + "parameterType": "com.acxiom.pipeline.connectors.DataConnector", + "description": "The data connector to use when writing", + "value": "!sourceBronzeConnector" + }, + { + "type": "text", + "name": "source", + "required": false, + "parameterType": "String", + "description": "The source path to load data", + "value": "!sourceBronzePath" + }, + { + "type": "object", + "name": "readOptions", + "required": false, + "className": "com.acxiom.pipeline.steps.DataFrameReaderOptions", + "parameterType": "com.acxiom.pipeline.steps.DataFrameReaderOptions", + "description": "The optional options to use while reading the data", + "value": "!sourceBronzeReadOptions" + } + ], + "tags": [ + "metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar" + ], + "type": "Pipeline", + "stepId": "836aab38-1140-4606-ab73-5b6744f0e7e7", + "nextStepId": "ExecuteColumnCleanup" + }, + { + "id": "ExecuteColumnCleanup", + "displayName": "String Equals", + "description": "Return whether string1 equals string2", + "type": "branch", + "params": [ + { + "type": "text", + "name": "string", + "required": true, + "value": "!executeColumnCleanup || true", + "description": "The string to compare" + }, + { + "type": "text", + "name": "anotherString", + "required": true, + "value": "true", + "description": "The other string to compare" + }, + { + "type": "boolean", + "name": "caseInsensitive", + "required": false, + "value": true, + "description": "Boolean flag to indicate case sensitive compare" + }, + { + "type": "result", + "name": "true", + "required": false, + "value": "StandardizeColumnNames", + "description": "" + }, + { + "type": "result", + "name": "false", + "required": false, + "value": "AddRecordIdDecision", + "description": "" + } + ], + "engineMeta": { + "spark": "StringSteps.stringEquals", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "Boolean" + } + }, + "stepId": "3fabf9ec-5383-4eb3-81af-6092ab7c370d" + }, + { + "id": "StandardizeColumnNames", + "displayName": "Standardize Column Names on a DataFrame", + "description": "This step will standardize columns names on existing dataframe", + "type": "Pipeline", + "params": [ + { + "type": "text", + "name": "dataFrame", + "required": false, + "value": "@Load", + "description": "" + } + ], + "engineMeta": { + "spark": "TransformationSteps.standardizeColumnNames", + "pkg": "com.acxiom.pipeline.steps" + }, + "nextStepId": "AddRecordIdDecision", + "stepId": "a981080d-714c-4d36-8b09-d95842ec5655" + }, + { + "id": "AddRecordIdDecision", + "displayName": "String Equals", + "description": "Return whether string1 equals string2", + "type": "branch", + "params": [ + { + "type": "text", + "name": "string", + "required": true, + "value": "!addRecordId || true", + "description": "The string to compare" + }, + { + "type": "text", + "name": "anotherString", + "required": true, + "value": "true", + "description": "The other string to compare" + }, + { + "type": "boolean", + "name": "caseInsensitive", + "required": false, + "value": true, + "description": "Boolean flag to indicate case sensitive compare" + }, + { + "type": "result", + "name": "true", + "required": false, + "value": "AddRecordId", + "description": "" + }, + { + "type": "result", + "name": "false", + "required": false, + "value": "AddFileIdDecision", + "description": "" + } + ], + "engineMeta": { + "spark": "StringSteps.stringEquals", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "Boolean" + } + }, + "stepId": "3fabf9ec-5383-4eb3-81af-6092ab7c370d" + }, + { + "id": "AddRecordId", + "displayName": "Adds a Unique Identifier to a DataFrame", + "description": "This step will add a new unique identifier to an existing data frame", + "type": "Pipeline", + "params": [ + { + "type": "text", + "name": "idColumnName", + "required": false, + "value": "metalus_record_id", + "description": "" + }, + { + "type": "text", + "name": "dataFrame", + "required": false, + "value": "@StandardizeColumnNames || @LoadDataFrame", + "description": "" + } + ], + "engineMeta": { + "spark": "DataSteps.addUniqueIdToDataFrame", + "pkg": "com.acxiom.pipeline.steps" + }, + "nextStepId": "AddFileIdDecision", + "stepId": "9f7d84b0-ebab-57da-8b39-be4c47028242" + }, + { + "id": "AddFileIdDecision", + "displayName": "String Equals", + "description": "Return whether string1 equals string2", + "type": "branch", + "params": [ + { + "type": "text", + "name": "string", + "required": true, + "value": "!addFileId || true", + "description": "The string to compare" + }, + { + "type": "text", + "name": "anotherString", + "required": true, + "value": "true", + "description": "The other string to compare" + }, + { + "type": "boolean", + "name": "caseInsensitive", + "required": false, + "value": true, + "description": "Boolean flag to indicate case sensitive compare" + }, + { + "type": "result", + "name": "true", + "required": false, + "value": "AddFileId", + "description": "" + }, + { + "type": "result", + "name": "false", + "required": false, + "value": "Write", + "description": "" + } + ], + "engineMeta": { + "spark": "StringSteps.stringEquals", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "Boolean" + } + }, + "stepId": "3fabf9ec-5383-4eb3-81af-6092ab7c370d" + }, + { + "id": "AddFileId", + "displayName": "Add a Column with a Static Value to All Rows in a DataFrame", + "description": "This step will add a column with a static value to all rows in the provided data frame", + "type": "Pipeline", + "params": [ + { + "type": "text", + "name": "dataFrame", + "required": false, + "value": "@AddRecordId || @StandardizeColumnNames || @LoadDataFrame", + "description": "" + }, + { + "type": "text", + "name": "columnName", + "required": false, + "value": "metalus_file_id", + "description": "" + }, + { + "type": "text", + "name": "columnValue", + "required": false, + "value": "!fileId", + "description": "" + } + ], + "engineMeta": { + "spark": "DataSteps.addStaticColumnToDataFrame", + "pkg": "com.acxiom.pipeline.steps" + }, + "nextStepId": "Write", + "stepId": "37e10488-02c1-5c85-b47a-efecf681fdd4" + }, + { + "id": "Write", + "category": "Connectors", + "creationDate": "2021-10-06T09:34:22.558Z", + "description": "This step will write a DataFrame using the given DataConnector", + "displayName": "Write", + "engineMeta": { + "spark": "DataConnectorSteps.writeDataFrame", + "pkg": "com.acxiom.pipeline.steps", + "results": { + "primaryType": "org.apache.spark.sql.streaming.StreamingQuery" + } + }, + "modifiedDate": "2021-10-07T11:41:28.969Z", + "params": [ + { + "type": "text", + "name": "dataFrame", + "required": true, + "parameterType": "org.apache.spark.sql.DataFrame", + "description": "The DataFrame to write", + "value": "@AddFileId || @AddRecordId || @StandardizeColumnNames || @Load" + }, + { + "type": "text", + "name": "connector", + "required": true, + "parameterType": "com.acxiom.pipeline.connectors.DataConnector", + "description": "The data connector to use when writing", + "value": "!destinationBronzeConnector" + }, + { + "type": "text", + "name": "destination", + "required": false, + "parameterType": "String", + "description": "The destination path to write data", + "value": "!destinationBronzePath" + }, + { + "type": "object", + "name": "writeOptions", + "required": false, + "className": "com.acxiom.pipeline.steps.DataFrameWriterOptions", + "parameterType": "com.acxiom.pipeline.steps.DataFrameWriterOptions", + "description": "The optional DataFrame options to use while writing", + "value": "!destinationBronzeWriteOptions" + } + ], + "tags": [ + "metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar" + ], + "type": "Pipeline", + "stepId": "5608eba7-e9ff-48e6-af77-b5e810b99d89" + } + ] +} From 9f931da0076f45eec9c53fd8971f3147305c3906 Mon Sep 17 00:00:00 2001 From: dafreels Date: Fri, 8 Oct 2021 09:04:04 -0400 Subject: [PATCH 19/24] #252 updated documentation --- metalus-common/readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metalus-common/readme.md b/metalus-common/readme.md index ee663adf..838e1fea 100644 --- a/metalus-common/readme.md +++ b/metalus-common/readme.md @@ -24,7 +24,8 @@ using Spark. * [TransformationSteps](docs/transformationsteps.md) ## Pipelines/Step Groups -* [Copy File](docs/copyfile.md) +* [Copy File](docs/copyfile.md) * _Uses new connectors api_ +* [Load To Bronze](docs/loadtobronze.md) * _Uses new connectors api_ * [SFTP to HDFS](docs/sftp2hdfs.md) * [Download to Bronze HDFS](docs/downloadToBronzeHdfs.md) * [LoadToParquet](docs/loadtoparquet.md) From 122ffc033e4225cf75a8eb35d50653d34f95f940 Mon Sep 17 00:00:00 2001 From: dafreels Date: Mon, 11 Oct 2021 07:49:56 -0400 Subject: [PATCH 20/24] #252 updated documentation --- .../testData/metalus-aws/pipelines.json | 2 +- .../testData/metalus-common/pipelines.json | 2 +- .../testData/metalus-common/steps.json | 2 +- metalus-common/docs/dataoptions.md | 53 +++++++++++++++++++ metalus-gcp/docs/gcssteps.md | 6 +-- 5 files changed, 59 insertions(+), 6 deletions(-) diff --git a/manual_tests/testData/metalus-aws/pipelines.json b/manual_tests/testData/metalus-aws/pipelines.json index d924343a..24e2d93d 100644 --- a/manual_tests/testData/metalus-aws/pipelines.json +++ b/manual_tests/testData/metalus-aws/pipelines.json @@ -1 +1 @@ -[{"id":"a8f1acd0-c39b-11eb-b944-4f8822efc9f5","name":"WriteDataFrameToS3","steps":[{"id":"Get_Credential","displayName":"Get Credential","description":"This step provides access to credentials through the CredentialProvider","type":"Pipeline","params":[{"type":"text","name":"credentialName","required":true,"value":"!credentialName","parameterType":"String","description":"The dataset containing CSV strings"}],"engineMeta":{"spark":"CredentialSteps.getCredential","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.Credential"}},"nextStepId":"WriteToParquetS3","stepId":"86c84fa3-ad45-4a49-ac05-92385b8e9572"},{"id":"WriteToParquetS3","displayName":"Write DataFrame to S3","description":"This step will write a DataFrame in a given format to S3","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":true,"value":"!inputDataFrame","description":"The DataFrame to post to the Kinesis stream"},{"type":"text","name":"path","required":true,"value":"!{bronzeZonePath}/!{fileId}","description":"The S3 path to write data"},{"type":"text","name":"accessKeyId","required":false,"value":"@Get_Credential.awsAccessKey","description":"The optional API key to use for S3 access"},{"type":"text","name":"secretAccessKey","required":false,"value":"@Get_Credential.awsAccessSecret","description":"The optional API secret to use for S3 access"},{"type":"object","name":"options","required":false,"value":{"format":"parquet","bucketingOptions":{},"options":{"escapeQuotes":false},"schema":{"attributes":[]},"saveMode":"Overwrite"},"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The optional DataFrame Options"}],"engineMeta":{"spark":"S3Steps.writeToPath","pkg":"com.acxiom.aws.steps"},"stepId":"7dc79901-795f-4610-973c-f46da63f669c"}],"category":"step-group","tags":["metalus-aws_2.11-spark_2.4-1.8.1.jar"]},{"id":"ff30fd80-c39b-11eb-b944-4f8822efc9f5","name":"LoadS3Data","steps":[{"id":"Get_Credentials","displayName":"Get Credential","description":"This step provides access to credentials through the CredentialProvider","type":"Pipeline","params":[{"type":"text","name":"credentialName","required":true,"value":"!credentialName","parameterType":"String","description":"The dataset containing CSV strings"}],"engineMeta":{"spark":"CredentialSteps.getCredential","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.Credential"}},"nextStepId":"LoadS3Data","stepId":"86c84fa3-ad45-4a49-ac05-92385b8e9572"},{"id":"LoadS3Data","displayName":"Load DataFrame from S3 path","description":"This step will read a DataFrame from the given S3 path","type":"Pipeline","params":[{"type":"text","name":"path","required":true,"value":"!{landingPath}/!{fileId}","description":"The S3 path to load data"},{"type":"text","name":"accessKeyId","required":false,"value":"@Get_Credentials.awsAccessKey","description":"The optional API key to use for S3 access"},{"type":"text","name":"secretAccessKey","required":false,"value":"@Get_Credentials.awsAccessSecret","description":"The optional API secret to use for S3 access"},{"type":"object","name":"options","required":false,"value":"!readOptions","className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The optional DataFrame Options"}],"engineMeta":{"spark":"S3Steps.readFromPath","pkg":"com.acxiom.aws.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"stepId":"bd4a944f-39ad-4b9c-8bf7-6d3c1f356510"}],"category":"step-group","tags":["metalus-aws_2.11-spark_2.4-1.8.1.jar"],"stepGroupResult":"@LoadS3Data"}] +[{"id":"a8f1acd0-c39b-11eb-b944-4f8822efc9f5","name":"WriteDataFrameToS3","steps":[{"id":"Get_Credential","displayName":"Get Credential","description":"This step provides access to credentials through the CredentialProvider","type":"Pipeline","params":[{"type":"text","name":"credentialName","required":true,"value":"!credentialName","parameterType":"String","description":"The dataset containing CSV strings"}],"engineMeta":{"spark":"CredentialSteps.getCredential","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.Credential"}},"nextStepId":"WriteToParquetS3","stepId":"86c84fa3-ad45-4a49-ac05-92385b8e9572","retryLimit":-1},{"id":"WriteToParquetS3","displayName":"Write DataFrame to S3","description":"This step will write a DataFrame in a given format to S3","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":true,"value":"!inputDataFrame","description":"The DataFrame to post to the Kinesis stream"},{"type":"text","name":"path","required":true,"value":"!{bronzeZonePath}/!{fileId}","description":"The S3 path to write data"},{"type":"text","name":"accessKeyId","required":false,"value":"@Get_Credential.awsAccessKey","description":"The optional API key to use for S3 access"},{"type":"text","name":"secretAccessKey","required":false,"value":"@Get_Credential.awsAccessSecret","description":"The optional API secret to use for S3 access"},{"type":"object","name":"options","required":false,"value":{"format":"parquet","bucketingOptions":{},"options":{"escapeQuotes":false},"schema":{"attributes":[]},"saveMode":"Overwrite"},"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The optional DataFrame Options"}],"engineMeta":{"spark":"S3Steps.writeToPath","pkg":"com.acxiom.aws.steps"},"stepId":"7dc79901-795f-4610-973c-f46da63f669c","retryLimit":-1}],"category":"step-group","tags":["metalus-aws_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"ff30fd80-c39b-11eb-b944-4f8822efc9f5","name":"LoadS3Data","steps":[{"id":"Get_Credentials","displayName":"Get Credential","description":"This step provides access to credentials through the CredentialProvider","type":"Pipeline","params":[{"type":"text","name":"credentialName","required":true,"value":"!credentialName","parameterType":"String","description":"The dataset containing CSV strings"}],"engineMeta":{"spark":"CredentialSteps.getCredential","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.Credential"}},"nextStepId":"LoadS3Data","stepId":"86c84fa3-ad45-4a49-ac05-92385b8e9572","retryLimit":-1},{"id":"LoadS3Data","displayName":"Load DataFrame from S3 path","description":"This step will read a DataFrame from the given S3 path","type":"Pipeline","params":[{"type":"text","name":"path","required":true,"value":"!{landingPath}/!{fileId}","description":"The S3 path to load data"},{"type":"text","name":"accessKeyId","required":false,"value":"@Get_Credentials.awsAccessKey","description":"The optional API key to use for S3 access"},{"type":"text","name":"secretAccessKey","required":false,"value":"@Get_Credentials.awsAccessSecret","description":"The optional API secret to use for S3 access"},{"type":"object","name":"options","required":false,"value":"!readOptions","className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The optional DataFrame Options"}],"engineMeta":{"spark":"S3Steps.readFromPath","pkg":"com.acxiom.aws.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"stepId":"bd4a944f-39ad-4b9c-8bf7-6d3c1f356510","retryLimit":-1}],"category":"step-group","tags":["metalus-aws_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"stepGroupResult":"@LoadS3Data"}] diff --git a/manual_tests/testData/metalus-common/pipelines.json b/manual_tests/testData/metalus-common/pipelines.json index 82413032..d7becc66 100644 --- a/manual_tests/testData/metalus-common/pipelines.json +++ b/manual_tests/testData/metalus-common/pipelines.json @@ -1 +1 @@ -[{"id":"f4835500-4c4a-11ea-9c79-f31d60741e3b","name":"DownloadToBronzeHdfs","steps":[{"id":"DownloadToHdfs","displayName":"Step Group","description":"Allows pipelines to be executed as a single step within a parent pipeline.","type":"step-group","params":[{"type":"text","name":"pipelineId","required":false,"value":"46f5e310-4c47-11ea-a0a7-a749c3ebbd62","description":""},{"type":"text","name":"pipeline","required":false,"value":"&46f5e310-4c47-11ea-a0a7-a749c3ebbd62","description":""},{"type":"object","name":"pipelineMappings","required":false,"value":{"fileId":"!fileId","output_buffer_size":"!outputBufferSize || 65536","input_buffer_size":"!inputBufferSize || 65536","sftp_port":"!sftpPort || 22","sftp_input_path":"!sftpInputPath","sftp_username":"!sftpUsername","landing_path":"!landingPath","sftp_password":"!sftpPassword","read_buffer_size":"!readBufferSize || 32768","sftp_host":"!sftpHost"},"description":""}],"nextStepId":"LandingFileToDataFrame","stepId":"f09b3b9c-82ac-56de-8dc8-f57c063dd4aa"},{"id":"LandingFileToDataFrame","displayName":"Load DataFrame from HDFS path","description":"This step will read a dataFrame from the given HDFS path","type":"Pipeline","params":[{"type":"text","name":"path","required":false,"value":"!{landingPath}/!{fileId}","description":""},{"type":"object","name":"options","required":false,"value":"!inputReaderOptions","className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":""}],"engineMeta":{"spark":"HDFSSteps.readFromPath","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"StandardizeColumnNames","stepId":"87db259d-606e-46eb-b723-82923349640f"},{"id":"StandardizeColumnNames","displayName":"Standardize Column Names on a DataFrame","description":"This step will standardize columns names on existing dataframe","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":false,"value":"@LandingFileToDataFrame","description":""}],"engineMeta":{"spark":"TransformationSteps.standardizeColumnNames","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"AddRecordId","stepId":"a981080d-714c-4d36-8b09-d95842ec5655"},{"id":"AddRecordId","displayName":"Adds a Unique Identifier to a DataFrame","description":"This step will add a new unique identifier to an existing data frame","type":"Pipeline","params":[{"type":"text","name":"idColumnName","required":false,"value":"metalus_record_id","description":""},{"type":"text","name":"dataFrame","required":false,"value":"@StandardizeColumnNames","description":""}],"engineMeta":{"spark":"DataSteps.addUniqueIdToDataFrame","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"AddFileId","stepId":"9f7d84b0-ebab-57da-8b39-be4c47028242"},{"id":"AddFileId","displayName":"Add a Column with a Static Value to All Rows in a DataFrame","description":"This step will add a column with a static value to all rows in the provided data frame","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":false,"value":"@AddRecordId","description":""},{"type":"text","name":"columnName","required":false,"value":"metalus_file_id","description":""},{"type":"text","name":"columnValue","required":false,"value":"!fileId","description":""}],"engineMeta":{"spark":"DataSteps.addStaticColumnToDataFrame","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"WriteToParquetHdfs","stepId":"37e10488-02c1-5c85-b47a-efecf681fdd4"},{"id":"WriteToParquetHdfs","displayName":"Write DataFrame to HDFS","description":"This step will write a dataFrame in a given format to HDFS","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":false,"value":"@AddFileId","description":""},{"type":"text","name":"path","required":false,"value":"!{bronzeZonePath}/!{fileId}","description":""},{"type":"object","name":"options","required":false,"value":{"format":"parquet","saveMode":"Overwrite","options":{},"schema":{"attributes":[]}},"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":""}],"engineMeta":{"spark":"HDFSSteps.writeToPath","pkg":"com.acxiom.pipeline.steps"},"stepId":"0a296858-e8b7-43dd-9f55-88d00a7cd8fa"}],"category":"pipeline","tags":["metalus-common_2.11-spark_2.4-1.8.1.jar"]},{"id":"dcff1d10-c2c3-11eb-928b-3dca5c59af1b","name":"LoadToParquet","steps":[{"id":"LoadDataFrame","displayName":"Step Group","description":"Allows pipelines to be executed as a single step within a parent pipeline.","type":"step-group","params":[{"type":"text","name":"pipelineId","required":false,"value":"!loadDataFramePipelineId","description":"The id of the pipeline to execute. Either this parameter or the pipeline parameter must be set."},{"type":"text","name":"pipeline","required":false,"description":"The pipeline to execute. Either this parameter or the pipelineId parameter must be set. This may be a mapped value or a pipeline object."},{"type":"boolean","name":"useParentGlobals","required":false,"value":true,"description":"Indicates whether the calling pipeline globals should be merged with the pipelineMappings."},{"type":"object","name":"pipelineMappings","required":false,"value":{},"description":"The values to use as the globals for the pipeline. Values may be mapped from the outer pipeline context."}],"nextStepId":"ExecuteColumnCleanup","stepId":"f09b3b9c-82ac-56de-8dc8-f57c063dd4aa"},{"id":"ExecuteColumnCleanup","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","params":[{"type":"text","name":"string","required":true,"value":"!executeColumnCleanup || true","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"value":"true","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"value":true,"description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false,"value":"StandardizeColumnNames","description":""},{"type":"result","name":"false","required":false,"value":"AddRecordIdDecision","description":""}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"stepId":"3fabf9ec-5383-4eb3-81af-6092ab7c370d"},{"id":"StandardizeColumnNames","displayName":"Standardize Column Names on a DataFrame","description":"This step will standardize columns names on existing dataframe","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":false,"value":"@LoadDataFrame","description":""}],"engineMeta":{"spark":"TransformationSteps.standardizeColumnNames","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"AddRecordIdDecision","stepId":"a981080d-714c-4d36-8b09-d95842ec5655"},{"id":"AddRecordIdDecision","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","params":[{"type":"text","name":"string","required":true,"value":"!addRecordId || true","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"value":"true","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"value":true,"description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false,"value":"AddRecordId","description":""},{"type":"result","name":"false","required":false,"value":"AddFileIdDecision","description":""}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"stepId":"3fabf9ec-5383-4eb3-81af-6092ab7c370d"},{"id":"AddRecordId","displayName":"Adds a Unique Identifier to a DataFrame","description":"This step will add a new unique identifier to an existing data frame","type":"Pipeline","params":[{"type":"text","name":"idColumnName","required":false,"value":"metalus_record_id","description":""},{"type":"text","name":"dataFrame","required":false,"value":"@StandardizeColumnNames || @LoadDataFrame","description":""}],"engineMeta":{"spark":"DataSteps.addUniqueIdToDataFrame","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"AddFileIdDecision","stepId":"9f7d84b0-ebab-57da-8b39-be4c47028242"},{"id":"AddFileIdDecision","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","params":[{"type":"text","name":"string","required":true,"value":"!addFileId || true","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"value":"true","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"value":true,"description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false,"value":"AddFileId","description":""},{"type":"result","name":"false","required":false,"value":"WriteDataFrame","description":""}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"stepId":"3fabf9ec-5383-4eb3-81af-6092ab7c370d"},{"id":"AddFileId","displayName":"Add a Column with a Static Value to All Rows in a DataFrame","description":"This step will add a column with a static value to all rows in the provided data frame","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":false,"value":"@AddRecordId || @StandardizeColumnNames || @LoadDataFrame","description":""},{"type":"text","name":"columnName","required":false,"value":"metalus_file_id","description":""},{"type":"text","name":"columnValue","required":false,"value":"!fileId","description":""}],"engineMeta":{"spark":"DataSteps.addStaticColumnToDataFrame","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"WriteDataFrame","stepId":"37e10488-02c1-5c85-b47a-efecf681fdd4"},{"id":"WriteDataFrame","displayName":"Step Group","description":"Allows pipelines to be executed as a single step within a parent pipeline.","type":"step-group","params":[{"type":"text","name":"pipelineId","required":false,"value":"!writeDataFramePipelineId","description":"The id of the pipeline to execute. Either this parameter or the pipeline parameter must be set."},{"type":"text","name":"pipeline","required":false,"description":"The pipeline to execute. Either this parameter or the pipelineId parameter must be set. This may be a mapped value or a pipeline object."},{"type":"boolean","name":"useParentGlobals","required":false,"value":true,"description":"Indicates whether the calling pipeline globals should be merged with the pipelineMappings."},{"type":"object","name":"pipelineMappings","required":false,"value":{"inputDataFrame":"@AddFileId || @AddRecordId || @StandardizeColumnNames || @LoadDataFrame"},"description":"The values to use as the globals for the pipeline. Values may be mapped from the outer pipeline context."}],"stepId":"f09b3b9c-82ac-56de-8dc8-f57c063dd4aa"}],"category":"pipeline","tags":["metalus-common_2.11-spark_2.4-1.8.1.jar"]},{"id":"189328f0-c2c7-11eb-928b-3dca5c59af1b","name":"WriteDataFrameToHDFS","steps":[{"id":"WriteToParquetHdfs","displayName":"Write DataFrame to HDFS","description":"This step will write a dataFrame in a given format to HDFS","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":true,"value":"!inputDataFrame","description":"The DataFrame to write"},{"type":"text","name":"path","required":true,"value":"!{bronzeZonePath}/!{fileId}","description":"The GCS path to write data"},{"type":"object","name":"options","required":false,"value":{"format":"parquet","bucketingOptions":{},"options":{"escapeQuotes":false},"schema":{"attributes":[]},"saveMode":"Overwrite"},"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The optional DataFrame Options"}],"engineMeta":{"spark":"HDFSSteps.writeToPath","pkg":"com.acxiom.pipeline.steps"},"stepId":"0a296858-e8b7-43dd-9f55-88d00a7cd8fa"}],"category":"step-group","tags":["metalus-common_2.11-spark_2.4-1.8.1.jar"]},{"id":"46f5e310-4c47-11ea-a0a7-a749c3ebbd62","name":"SG_SftpToHdfs","steps":[{"id":"CreateSFTPFileManager","displayName":"Create SFTP FileManager","description":"Simple function to generate the SFTPFileManager for the remote SFTP file system","type":"Pipeline","params":[{"type":"text","name":"hostName","required":false,"value":"!sftp_host","description":""},{"type":"text","name":"username","required":false,"value":"!sftp_username","description":""},{"type":"text","name":"password","required":false,"value":"!sftp_password","description":""},{"type":"integer","name":"port","required":false,"value":"!sftp_port || 22","description":""},{"type":"text","name":"strictHostChecking","required":false,"value":false,"description":""}],"engineMeta":{"spark":"SFTPSteps.createFileManager","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"CreateHDFSFileManager","stepId":"9d467cb0-8b3d-40a0-9ccd-9cf8c5b6cb38"},{"id":"CreateHDFSFileManager","displayName":"Create HDFS FileManager","description":"Simple function to generate the HDFSFileManager for the local HDFS file system","type":"Pipeline","params":[],"engineMeta":{"spark":"HDFSSteps.createFileManager","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"DownloadFile","stepId":"e4dad367-a506-5afd-86c0-82c2cf5cd15c"},{"id":"DownloadFile","displayName":"Buffered file copy","description":"Copy the contents of the source path to the destination path using full buffer sizes. This function will call connect on both FileManagers.","type":"Pipeline","params":[{"type":"text","name":"srcFS","required":false,"value":"@CreateSFTPFileManager","description":""},{"type":"text","name":"srcPath","required":false,"value":"!sftp_input_path","description":""},{"type":"text","name":"destFS","required":false,"value":"@CreateHDFSFileManager","description":""},{"type":"text","name":"destPath","required":false,"value":"!{landing_path}/!{fileId}","description":""},{"type":"text","name":"inputBufferSize","required":false,"value":"!input_buffer_size || 65536","description":""},{"type":"text","name":"outputBufferSize","required":false,"value":"!output_buffer_size || 65536","description":""},{"type":"text","name":"copyBufferSize","required":false,"value":"!read_buffer_size || 32768","description":""}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"DisconnectSFTPFileManager","stepId":"f5a24db0-e91b-5c88-8e67-ab5cff09c883"},{"id":"DisconnectSFTPFileManager","displayName":"Disconnect a FileManager","description":"Disconnects a FileManager from the underlying file system","type":"Pipeline","params":[{"type":"text","name":"fileManager","required":false,"value":"@CreateSFTPFileManager","description":""}],"engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"stepId":"3d1e8519-690c-55f0-bd05-1e7b97fb6633"}],"category":"step-group","tags":["metalus-common_2.11-spark_2.4-1.8.1.jar"]},{"id":"e9ce4710-beda-11eb-977b-1f7c49e5a75d","name":"DownloadSFTPToHDFSWithDataFrame","steps":[{"id":"CreateSFTPFileManager","displayName":"Create SFTP FileManager","description":"Simple function to generate the SFTPFileManager for the remote SFTP file system","type":"Pipeline","params":[{"type":"text","name":"hostName","required":false,"value":"!sftp_host","description":""},{"type":"text","name":"username","required":false,"value":"!sftp_username","description":""},{"type":"text","name":"password","required":false,"value":"!sftp_password","description":""},{"type":"integer","name":"port","required":false,"value":"!sftp_port || 22","description":""},{"type":"text","name":"strictHostChecking","required":false,"value":false,"description":""}],"engineMeta":{"spark":"SFTPSteps.createFileManager","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"CreateHDFSFileManager","stepId":"9d467cb0-8b3d-40a0-9ccd-9cf8c5b6cb38"},{"id":"CreateHDFSFileManager","displayName":"Create HDFS FileManager","description":"Simple function to generate the HDFSFileManager for the local HDFS file system","type":"Pipeline","params":[],"engineMeta":{"spark":"HDFSSteps.createFileManager","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"DownloadFile","stepId":"e4dad367-a506-5afd-86c0-82c2cf5cd15c"},{"id":"DownloadFile","displayName":"Buffered file copy","description":"Copy the contents of the source path to the destination path using full buffer sizes. This function will call connect on both FileManagers.","type":"Pipeline","params":[{"type":"text","name":"srcFS","required":false,"value":"@CreateSFTPFileManager","description":""},{"type":"text","name":"srcPath","required":false,"value":"!sftp_input_path","description":""},{"type":"text","name":"destFS","required":false,"value":"@CreateHDFSFileManager","description":""},{"type":"text","name":"destPath","required":false,"value":"!{landing_path}/!{fileId}","description":""},{"type":"text","name":"inputBufferSize","required":false,"value":"!input_buffer_size || 65536","description":""},{"type":"text","name":"outputBufferSize","required":false,"value":"!output_buffer_size || 65536","description":""},{"type":"text","name":"copyBufferSize","required":false,"value":"!read_buffer_size || 32768","description":""}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"DisconnectSFTPFileManager","stepId":"f5a24db0-e91b-5c88-8e67-ab5cff09c883"},{"id":"DisconnectSFTPFileManager","displayName":"Disconnect a FileManager","description":"Disconnects a FileManager from the underlying file system","type":"Pipeline","params":[{"type":"text","name":"fileManager","required":false,"value":"@CreateSFTPFileManager","description":""}],"engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"LandingFileToDataFrame","stepId":"3d1e8519-690c-55f0-bd05-1e7b97fb6633"},{"id":"LandingFileToDataFrame","displayName":"Load DataFrame from HDFS path","description":"This step will read a dataFrame from the given HDFS path","type":"Pipeline","params":[{"type":"text","name":"path","required":true,"value":"!{landingPath}/!{fileId}","description":"The HDFS path to load data into the DataFrame"},{"type":"object","name":"options","required":false,"value":"!readOptions","className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"HDFSSteps.readFromPath","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"stepId":"87db259d-606e-46eb-b723-82923349640f"}],"category":"step-group","tags":["metalus-common_2.11-spark_2.4-1.8.1.jar"],"stepGroupResult":"@LandingFileToDataFrame"}] +[{"id":"f4835500-4c4a-11ea-9c79-f31d60741e3b","name":"DownloadToBronzeHdfs","steps":[{"id":"DownloadToHdfs","displayName":"Step Group","description":"Allows pipelines to be executed as a single step within a parent pipeline.","type":"step-group","params":[{"type":"text","name":"pipelineId","required":false,"value":"46f5e310-4c47-11ea-a0a7-a749c3ebbd62","description":""},{"type":"text","name":"pipeline","required":false,"value":"&46f5e310-4c47-11ea-a0a7-a749c3ebbd62","description":""},{"type":"object","name":"pipelineMappings","required":false,"value":{"fileId":"!fileId","output_buffer_size":"!outputBufferSize || 65536","input_buffer_size":"!inputBufferSize || 65536","sftp_port":"!sftpPort || 22","sftp_input_path":"!sftpInputPath","sftp_username":"!sftpUsername","landing_path":"!landingPath","sftp_password":"!sftpPassword","read_buffer_size":"!readBufferSize || 32768","sftp_host":"!sftpHost"},"description":""}],"nextStepId":"LandingFileToDataFrame","stepId":"f09b3b9c-82ac-56de-8dc8-f57c063dd4aa","retryLimit":-1},{"id":"LandingFileToDataFrame","displayName":"Load DataFrame from HDFS path","description":"This step will read a dataFrame from the given HDFS path","type":"Pipeline","params":[{"type":"text","name":"path","required":false,"value":"!{landingPath}/!{fileId}","description":""},{"type":"object","name":"options","required":false,"value":"!inputReaderOptions","className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":""}],"engineMeta":{"spark":"HDFSSteps.readFromPath","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"StandardizeColumnNames","stepId":"87db259d-606e-46eb-b723-82923349640f","retryLimit":-1},{"id":"StandardizeColumnNames","displayName":"Standardize Column Names on a DataFrame","description":"This step will standardize columns names on existing dataframe","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":false,"value":"@LandingFileToDataFrame","description":""}],"engineMeta":{"spark":"TransformationSteps.standardizeColumnNames","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"AddRecordId","stepId":"a981080d-714c-4d36-8b09-d95842ec5655","retryLimit":-1},{"id":"AddRecordId","displayName":"Adds a Unique Identifier to a DataFrame","description":"This step will add a new unique identifier to an existing data frame","type":"Pipeline","params":[{"type":"text","name":"idColumnName","required":false,"value":"metalus_record_id","description":""},{"type":"text","name":"dataFrame","required":false,"value":"@StandardizeColumnNames","description":""}],"engineMeta":{"spark":"DataSteps.addUniqueIdToDataFrame","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"AddFileId","stepId":"9f7d84b0-ebab-57da-8b39-be4c47028242","retryLimit":-1},{"id":"AddFileId","displayName":"Add a Column with a Static Value to All Rows in a DataFrame","description":"This step will add a column with a static value to all rows in the provided data frame","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":false,"value":"@AddRecordId","description":""},{"type":"text","name":"columnName","required":false,"value":"metalus_file_id","description":""},{"type":"text","name":"columnValue","required":false,"value":"!fileId","description":""}],"engineMeta":{"spark":"DataSteps.addStaticColumnToDataFrame","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"WriteToParquetHdfs","stepId":"37e10488-02c1-5c85-b47a-efecf681fdd4","retryLimit":-1},{"id":"WriteToParquetHdfs","displayName":"Write DataFrame to HDFS","description":"This step will write a dataFrame in a given format to HDFS","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":false,"value":"@AddFileId","description":""},{"type":"text","name":"path","required":false,"value":"!{bronzeZonePath}/!{fileId}","description":""},{"type":"object","name":"options","required":false,"value":{"format":"parquet","saveMode":"Overwrite","options":{},"schema":{"attributes":[]}},"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":""}],"engineMeta":{"spark":"HDFSSteps.writeToPath","pkg":"com.acxiom.pipeline.steps"},"stepId":"0a296858-e8b7-43dd-9f55-88d00a7cd8fa","retryLimit":-1}],"category":"pipeline","tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"dcff1d10-c2c3-11eb-928b-3dca5c59af1b","name":"LoadToParquet","steps":[{"id":"LoadDataFrame","displayName":"Step Group","description":"Allows pipelines to be executed as a single step within a parent pipeline.","type":"step-group","params":[{"type":"text","name":"pipelineId","required":false,"value":"!loadDataFramePipelineId","description":"The id of the pipeline to execute. Either this parameter or the pipeline parameter must be set."},{"type":"text","name":"pipeline","required":false,"description":"The pipeline to execute. Either this parameter or the pipelineId parameter must be set. This may be a mapped value or a pipeline object."},{"type":"boolean","name":"useParentGlobals","required":false,"value":true,"description":"Indicates whether the calling pipeline globals should be merged with the pipelineMappings."},{"type":"object","name":"pipelineMappings","required":false,"value":{},"description":"The values to use as the globals for the pipeline. Values may be mapped from the outer pipeline context."}],"nextStepId":"ExecuteColumnCleanup","stepId":"f09b3b9c-82ac-56de-8dc8-f57c063dd4aa","retryLimit":-1},{"id":"ExecuteColumnCleanup","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","params":[{"type":"text","name":"string","required":true,"value":"!executeColumnCleanup || true","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"value":"true","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"value":true,"description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false,"value":"StandardizeColumnNames","description":""},{"type":"result","name":"false","required":false,"value":"AddRecordIdDecision","description":""}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"stepId":"3fabf9ec-5383-4eb3-81af-6092ab7c370d","retryLimit":-1},{"id":"StandardizeColumnNames","displayName":"Standardize Column Names on a DataFrame","description":"This step will standardize columns names on existing dataframe","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":false,"value":"@LoadDataFrame","description":""}],"engineMeta":{"spark":"TransformationSteps.standardizeColumnNames","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"AddRecordIdDecision","stepId":"a981080d-714c-4d36-8b09-d95842ec5655","retryLimit":-1},{"id":"AddRecordIdDecision","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","params":[{"type":"text","name":"string","required":true,"value":"!addRecordId || true","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"value":"true","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"value":true,"description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false,"value":"AddRecordId","description":""},{"type":"result","name":"false","required":false,"value":"AddFileIdDecision","description":""}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"stepId":"3fabf9ec-5383-4eb3-81af-6092ab7c370d","retryLimit":-1},{"id":"AddRecordId","displayName":"Adds a Unique Identifier to a DataFrame","description":"This step will add a new unique identifier to an existing data frame","type":"Pipeline","params":[{"type":"text","name":"idColumnName","required":false,"value":"metalus_record_id","description":""},{"type":"text","name":"dataFrame","required":false,"value":"@StandardizeColumnNames || @LoadDataFrame","description":""}],"engineMeta":{"spark":"DataSteps.addUniqueIdToDataFrame","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"AddFileIdDecision","stepId":"9f7d84b0-ebab-57da-8b39-be4c47028242","retryLimit":-1},{"id":"AddFileIdDecision","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","params":[{"type":"text","name":"string","required":true,"value":"!addFileId || true","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"value":"true","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"value":true,"description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false,"value":"AddFileId","description":""},{"type":"result","name":"false","required":false,"value":"WriteDataFrame","description":""}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"stepId":"3fabf9ec-5383-4eb3-81af-6092ab7c370d","retryLimit":-1},{"id":"AddFileId","displayName":"Add a Column with a Static Value to All Rows in a DataFrame","description":"This step will add a column with a static value to all rows in the provided data frame","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":false,"value":"@AddRecordId || @StandardizeColumnNames || @LoadDataFrame","description":""},{"type":"text","name":"columnName","required":false,"value":"metalus_file_id","description":""},{"type":"text","name":"columnValue","required":false,"value":"!fileId","description":""}],"engineMeta":{"spark":"DataSteps.addStaticColumnToDataFrame","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"WriteDataFrame","stepId":"37e10488-02c1-5c85-b47a-efecf681fdd4","retryLimit":-1},{"id":"WriteDataFrame","displayName":"Step Group","description":"Allows pipelines to be executed as a single step within a parent pipeline.","type":"step-group","params":[{"type":"text","name":"pipelineId","required":false,"value":"!writeDataFramePipelineId","description":"The id of the pipeline to execute. Either this parameter or the pipeline parameter must be set."},{"type":"text","name":"pipeline","required":false,"description":"The pipeline to execute. Either this parameter or the pipelineId parameter must be set. This may be a mapped value or a pipeline object."},{"type":"boolean","name":"useParentGlobals","required":false,"value":true,"description":"Indicates whether the calling pipeline globals should be merged with the pipelineMappings."},{"type":"object","name":"pipelineMappings","required":false,"value":{"inputDataFrame":"@AddFileId || @AddRecordId || @StandardizeColumnNames || @LoadDataFrame"},"description":"The values to use as the globals for the pipeline. Values may be mapped from the outer pipeline context."}],"stepId":"f09b3b9c-82ac-56de-8dc8-f57c063dd4aa","retryLimit":-1}],"category":"pipeline","tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"189328f0-c2c7-11eb-928b-3dca5c59af1b","name":"WriteDataFrameToHDFS","steps":[{"id":"WriteToParquetHdfs","displayName":"Write DataFrame to HDFS","description":"This step will write a dataFrame in a given format to HDFS","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":true,"value":"!inputDataFrame","description":"The DataFrame to write"},{"type":"text","name":"path","required":true,"value":"!{bronzeZonePath}/!{fileId}","description":"The GCS path to write data"},{"type":"object","name":"options","required":false,"value":{"format":"parquet","bucketingOptions":{},"options":{"escapeQuotes":false},"schema":{"attributes":[]},"saveMode":"Overwrite"},"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The optional DataFrame Options"}],"engineMeta":{"spark":"HDFSSteps.writeToPath","pkg":"com.acxiom.pipeline.steps"},"stepId":"0a296858-e8b7-43dd-9f55-88d00a7cd8fa","retryLimit":-1}],"category":"step-group","tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"46f5e310-4c47-11ea-a0a7-a749c3ebbd62","name":"SG_SftpToHdfs","steps":[{"id":"CreateSFTPFileManager","displayName":"Create SFTP FileManager","description":"Simple function to generate the SFTPFileManager for the remote SFTP file system","type":"Pipeline","params":[{"type":"text","name":"hostName","required":false,"value":"!sftp_host","description":""},{"type":"text","name":"username","required":false,"value":"!sftp_username","description":""},{"type":"text","name":"password","required":false,"value":"!sftp_password","description":""},{"type":"integer","name":"port","required":false,"value":"!sftp_port || 22","description":""},{"type":"text","name":"strictHostChecking","required":false,"value":false,"description":""}],"engineMeta":{"spark":"SFTPSteps.createFileManager","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"CreateHDFSFileManager","stepId":"9d467cb0-8b3d-40a0-9ccd-9cf8c5b6cb38","retryLimit":-1},{"id":"CreateHDFSFileManager","displayName":"Create HDFS FileManager","description":"Simple function to generate the HDFSFileManager for the local HDFS file system","type":"Pipeline","params":[],"engineMeta":{"spark":"HDFSSteps.createFileManager","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"DownloadFile","stepId":"e4dad367-a506-5afd-86c0-82c2cf5cd15c","retryLimit":-1},{"id":"DownloadFile","displayName":"Buffered file copy","description":"Copy the contents of the source path to the destination path using full buffer sizes. This function will call connect on both FileManagers.","type":"Pipeline","params":[{"type":"text","name":"srcFS","required":false,"value":"@CreateSFTPFileManager","description":""},{"type":"text","name":"srcPath","required":false,"value":"!sftp_input_path","description":""},{"type":"text","name":"destFS","required":false,"value":"@CreateHDFSFileManager","description":""},{"type":"text","name":"destPath","required":false,"value":"!{landing_path}/!{fileId}","description":""},{"type":"text","name":"inputBufferSize","required":false,"value":"!input_buffer_size || 65536","description":""},{"type":"text","name":"outputBufferSize","required":false,"value":"!output_buffer_size || 65536","description":""},{"type":"text","name":"copyBufferSize","required":false,"value":"!read_buffer_size || 32768","description":""}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"DisconnectSFTPFileManager","stepId":"f5a24db0-e91b-5c88-8e67-ab5cff09c883","retryLimit":-1},{"id":"DisconnectSFTPFileManager","displayName":"Disconnect a FileManager","description":"Disconnects a FileManager from the underlying file system","type":"Pipeline","params":[{"type":"text","name":"fileManager","required":false,"value":"@CreateSFTPFileManager","description":""}],"engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"stepId":"3d1e8519-690c-55f0-bd05-1e7b97fb6633","retryLimit":-1}],"category":"step-group","tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"a9f62840-2827-11ec-9c0c-cbf3549779e5","name":"LoadToBronze","steps":[{"id":"Load","displayName":"Load","description":"This step will create a DataFrame using the given DataConnector","type":"Pipeline","params":[{"type":"text","name":"connector","required":true,"value":"!sourceBronzeConnector","parameterType":"com.acxiom.pipeline.connectors.DataConnector","description":"The data connector to use when writing"},{"type":"text","name":"source","required":false,"value":"!sourceBronzePath","parameterType":"String","description":"The source path to load data"},{"type":"object","name":"readOptions","required":false,"value":"!sourceBronzeReadOptions","className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The optional options to use while reading the data"}],"engineMeta":{"spark":"DataConnectorSteps.loadDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"nextStepId":"ExecuteColumnCleanup","stepId":"836aab38-1140-4606-ab73-5b6744f0e7e7","retryLimit":-1},{"id":"ExecuteColumnCleanup","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","params":[{"type":"text","name":"string","required":true,"value":"!executeColumnCleanup || true","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"value":"true","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"value":true,"description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false,"value":"StandardizeColumnNames","description":""},{"type":"result","name":"false","required":false,"value":"AddRecordIdDecision","description":""}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"stepId":"3fabf9ec-5383-4eb3-81af-6092ab7c370d","retryLimit":-1},{"id":"StandardizeColumnNames","displayName":"Standardize Column Names on a DataFrame","description":"This step will standardize columns names on existing dataframe","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":false,"value":"@Load","description":""}],"engineMeta":{"spark":"TransformationSteps.standardizeColumnNames","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"AddRecordIdDecision","stepId":"a981080d-714c-4d36-8b09-d95842ec5655","retryLimit":-1},{"id":"AddRecordIdDecision","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","params":[{"type":"text","name":"string","required":true,"value":"!addRecordId || true","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"value":"true","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"value":true,"description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false,"value":"AddRecordId","description":""},{"type":"result","name":"false","required":false,"value":"AddFileIdDecision","description":""}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"stepId":"3fabf9ec-5383-4eb3-81af-6092ab7c370d","retryLimit":-1},{"id":"AddRecordId","displayName":"Adds a Unique Identifier to a DataFrame","description":"This step will add a new unique identifier to an existing data frame","type":"Pipeline","params":[{"type":"text","name":"idColumnName","required":false,"value":"metalus_record_id","description":""},{"type":"text","name":"dataFrame","required":false,"value":"@StandardizeColumnNames || @LoadDataFrame","description":""}],"engineMeta":{"spark":"DataSteps.addUniqueIdToDataFrame","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"AddFileIdDecision","stepId":"9f7d84b0-ebab-57da-8b39-be4c47028242","retryLimit":-1},{"id":"AddFileIdDecision","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","params":[{"type":"text","name":"string","required":true,"value":"!addFileId || true","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"value":"true","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"value":true,"description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false,"value":"AddFileId","description":""},{"type":"result","name":"false","required":false,"value":"Write","description":""}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"stepId":"3fabf9ec-5383-4eb3-81af-6092ab7c370d","retryLimit":-1},{"id":"AddFileId","displayName":"Add a Column with a Static Value to All Rows in a DataFrame","description":"This step will add a column with a static value to all rows in the provided data frame","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":false,"value":"@AddRecordId || @StandardizeColumnNames || @LoadDataFrame","description":""},{"type":"text","name":"columnName","required":false,"value":"metalus_file_id","description":""},{"type":"text","name":"columnValue","required":false,"value":"!fileId","description":""}],"engineMeta":{"spark":"DataSteps.addStaticColumnToDataFrame","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"Write","stepId":"37e10488-02c1-5c85-b47a-efecf681fdd4","retryLimit":-1},{"id":"Write","displayName":"Write","description":"This step will write a DataFrame using the given DataConnector","type":"Pipeline","params":[{"type":"text","name":"dataFrame","required":true,"value":"@AddFileId || @AddRecordId || @StandardizeColumnNames || @Load","parameterType":"org.apache.spark.sql.DataFrame","description":"The DataFrame to write"},{"type":"text","name":"connector","required":true,"value":"!destinationBronzeConnector","parameterType":"com.acxiom.pipeline.connectors.DataConnector","description":"The data connector to use when writing"},{"type":"text","name":"destination","required":false,"value":"!destinationBronzePath","parameterType":"String","description":"The destination path to write data"},{"type":"object","name":"writeOptions","required":false,"value":"!destinationBronzeWriteOptions","className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The optional DataFrame options to use while writing"}],"engineMeta":{"spark":"DataConnectorSteps.writeDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.streaming.StreamingQuery"}},"stepId":"5608eba7-e9ff-48e6-af77-b5e810b99d89","retryLimit":-1}],"category":"pipeline","tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"e9ce4710-beda-11eb-977b-1f7c49e5a75d","name":"DownloadSFTPToHDFSWithDataFrame","steps":[{"id":"CreateSFTPFileManager","displayName":"Create SFTP FileManager","description":"Simple function to generate the SFTPFileManager for the remote SFTP file system","type":"Pipeline","params":[{"type":"text","name":"hostName","required":false,"value":"!sftp_host","description":""},{"type":"text","name":"username","required":false,"value":"!sftp_username","description":""},{"type":"text","name":"password","required":false,"value":"!sftp_password","description":""},{"type":"integer","name":"port","required":false,"value":"!sftp_port || 22","description":""},{"type":"text","name":"strictHostChecking","required":false,"value":false,"description":""}],"engineMeta":{"spark":"SFTPSteps.createFileManager","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"CreateHDFSFileManager","stepId":"9d467cb0-8b3d-40a0-9ccd-9cf8c5b6cb38","retryLimit":-1},{"id":"CreateHDFSFileManager","displayName":"Create HDFS FileManager","description":"Simple function to generate the HDFSFileManager for the local HDFS file system","type":"Pipeline","params":[],"engineMeta":{"spark":"HDFSSteps.createFileManager","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"DownloadFile","stepId":"e4dad367-a506-5afd-86c0-82c2cf5cd15c","retryLimit":-1},{"id":"DownloadFile","displayName":"Buffered file copy","description":"Copy the contents of the source path to the destination path using full buffer sizes. This function will call connect on both FileManagers.","type":"Pipeline","params":[{"type":"text","name":"srcFS","required":false,"value":"@CreateSFTPFileManager","description":""},{"type":"text","name":"srcPath","required":false,"value":"!sftp_input_path","description":""},{"type":"text","name":"destFS","required":false,"value":"@CreateHDFSFileManager","description":""},{"type":"text","name":"destPath","required":false,"value":"!{landing_path}/!{fileId}","description":""},{"type":"text","name":"inputBufferSize","required":false,"value":"!input_buffer_size || 65536","description":""},{"type":"text","name":"outputBufferSize","required":false,"value":"!output_buffer_size || 65536","description":""},{"type":"text","name":"copyBufferSize","required":false,"value":"!read_buffer_size || 32768","description":""}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"DisconnectSFTPFileManager","stepId":"f5a24db0-e91b-5c88-8e67-ab5cff09c883","retryLimit":-1},{"id":"DisconnectSFTPFileManager","displayName":"Disconnect a FileManager","description":"Disconnects a FileManager from the underlying file system","type":"Pipeline","params":[{"type":"text","name":"fileManager","required":false,"value":"@CreateSFTPFileManager","description":""}],"engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"LandingFileToDataFrame","stepId":"3d1e8519-690c-55f0-bd05-1e7b97fb6633","retryLimit":-1},{"id":"LandingFileToDataFrame","displayName":"Load DataFrame from HDFS path","description":"This step will read a dataFrame from the given HDFS path","type":"Pipeline","params":[{"type":"text","name":"path","required":true,"value":"!{landingPath}/!{fileId}","description":"The HDFS path to load data into the DataFrame"},{"type":"object","name":"options","required":false,"value":"!readOptions","className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"HDFSSteps.readFromPath","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"stepId":"87db259d-606e-46eb-b723-82923349640f","retryLimit":-1}],"category":"step-group","tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"],"stepGroupResult":"@LandingFileToDataFrame"},{"id":"43bc9450-2689-11ec-9c0c-cbf3549779e5","name":"CopyFile","steps":[{"id":"GETSOURCE","displayName":"Create a FileManager","description":"Creates a FileManager using the provided FileConnector","type":"Pipeline","params":[{"type":"text","name":"fileConnector","required":true,"value":"!sourceConnector","parameterType":"com.acxiom.pipeline.connectors.FileConnector","description":"The FileConnector to use to create the FileManager implementation"}],"engineMeta":{"spark":"FileManagerSteps.getFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.FileManager"}},"nextStepId":"GETDESTINATION","stepId":"259a880a-3e12-4843-9f02-2cfc2a05f576","retryLimit":-1},{"id":"GETDESTINATION","displayName":"Create a FileManager","description":"Creates a FileManager using the provided FileConnector","type":"Pipeline","params":[{"type":"text","name":"fileConnector","required":true,"value":"!destinationConnector","parameterType":"com.acxiom.pipeline.connectors.FileConnector","description":"The FileConnector to use to create the FileManager implementation"}],"engineMeta":{"spark":"FileManagerSteps.getFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.FileManager"}},"nextStepId":"COPY","stepId":"259a880a-3e12-4843-9f02-2cfc2a05f576","retryLimit":-1},{"id":"COPY","displayName":"Copy (auto buffering)","description":"Copy the contents of the source path to the destination path. This function will call connect on both FileManagers.","type":"Pipeline","params":[{"type":"text","name":"srcFS","required":true,"value":"@GETSOURCE","parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"value":"!sourceCopyPath","parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"value":"@GETDESTINATION","parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"value":"!destinationCopyPath","parameterType":"String","description":"The path to copy to"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}},"nextStepId":"VERIFY","stepId":"0342654c-2722-56fe-ba22-e342169545af","nextStepOnError":"DELETEDESTINATION","retryLimit":5},{"id":"VERIFY","displayName":"Compare File Sizes","description":"Compare the file sizes of the source and destination paths","type":"Pipeline","params":[{"type":"text","name":"srcFS","required":true,"value":"@GETSOURCE","parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"value":"!sourceCopyPath","parameterType":"String","description":"The path to the source"},{"type":"text","name":"destFS","required":true,"value":"@GETDESTINATION","parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"value":"!destinationCopyPath","parameterType":"String","description":"The path to th destination"}],"engineMeta":{"spark":"FileManagerSteps.compareFileSizes","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Int"}},"nextStepId":"TO_STRING","stepId":"1af68ab5-a3fe-4afb-b5fa-34e52f7c77f5","retryLimit":-1},{"id":"TO_STRING","displayName":"To String","description":"Returns the result of the toString method, can unwrap options","type":"Pipeline","params":[{"type":"text","name":"value","required":true,"value":"@VERIFY","parameterType":"Any","description":"The value to convert"},{"type":"boolean","name":"unwrapOption","required":false,"value":false,"parameterType":"Boolean","description":"Boolean indicating whether to unwrap the value from an Option prior to calling toString"}],"engineMeta":{"spark":"StringSteps.toString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"nextStepId":"CHECKRESULTS","stepId":"b5485d97-d4e8-41a6-8af7-9ce79a435140","retryLimit":-1},{"id":"CHECKRESULTS","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","params":[{"type":"text","name":"string","required":true,"value":"@TO_STRING","parameterType":"String","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"value":"0","parameterType":"String","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"value":false,"parameterType":"Boolean","description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false,"value":"CLOSESOURCE","description":""},{"type":"result","name":"false","required":false,"value":"DELETEDESTINATION","description":""}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"stepId":"3fabf9ec-5383-4eb3-81af-6092ab7c370d","retryLimit":-1},{"id":"CLOSESOURCE","displayName":"Disconnect a FileManager","description":"Disconnects a FileManager from the underlying file system","type":"Pipeline","params":[{"type":"text","name":"fileManager","required":true,"value":"@GETSOURCE","parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The file manager to disconnect"}],"engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"nextStepId":"CLOSEDESTINATION","stepId":"3d1e8519-690c-55f0-bd05-1e7b97fb6633","retryLimit":-1},{"id":"CLOSEDESTINATION","displayName":"Disconnect a FileManager","description":"Disconnects a FileManager from the underlying file system","type":"Pipeline","params":[{"type":"text","name":"fileManager","required":true,"value":"@GETDESTINATION","parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The file manager to disconnect"}],"engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"stepId":"3d1e8519-690c-55f0-bd05-1e7b97fb6633","retryLimit":-1},{"id":"DELETEDESTINATION","displayName":"Delete (file)","description":"Delete a file","type":"Pipeline","params":[{"type":"text","name":"fileManager","required":true,"value":"@GETDESTINATION","parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The FileManager"},{"type":"text","name":"path","required":true,"value":"!destinationCopyPath","parameterType":"String","description":"The path to the file being deleted"}],"engineMeta":{"spark":"FileManagerSteps.deleteFile","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"nextStepId":"RETRY","stepId":"bf2c4df8-a215-480b-87d8-586984e04189","retryLimit":-1},{"id":"RETRY","displayName":"Retry (simple)","description":"Makes a decision to retry or stop based on a named counter","type":"branch","params":[{"type":"text","name":"counterName","required":true,"value":"COPY_FILE_RETRY","parameterType":"String","description":"The name of the counter to use for tracking"},{"type":"integer","name":"maxRetries","required":true,"value":5,"parameterType":"Int","description":"The maximum number of retries allowed"},{"type":"result","name":"retry","required":false,"value":"COPY","description":""},{"type":"result","name":"stop","required":false,"value":"CLOSESOURCE","description":""}],"engineMeta":{"spark":"FlowUtilsSteps.simpleRetry","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"stepId":"6ed36f89-35d1-4280-a555-fbcd8dd76bf2","retryLimit":-1}],"category":"pipeline","tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]}] diff --git a/manual_tests/testData/metalus-common/steps.json b/manual_tests/testData/metalus-common/steps.json index 9f824fca..f400fd4f 100644 --- a/manual_tests/testData/metalus-common/steps.json +++ b/manual_tests/testData/metalus-common/steps.json @@ -1 +1 @@ -{"pkgs":["com.acxiom.pipeline.steps"],"steps":[{"id":"3806f23b-478c-4054-b6c1-37f11db58d38","displayName":"Read a DataFrame from Table","description":"This step will read a dataFrame in a given format from the meta store","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"table","required":true,"parameterType":"String","description":"The name of the table to read"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The DataFrameReaderOptions to use"}],"engineMeta":{"spark":"CatalogSteps.readDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"e2b4c011-e71b-46f9-a8be-cf937abc2ec4","displayName":"Write DataFrame to Table","description":"This step will write a dataFrame in a given format to the meta store","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to write"},{"type":"text","name":"table","required":true,"parameterType":"String","description":"The name of the table to write to"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The DataFrameWriterOptions to use"}],"engineMeta":{"spark":"CatalogSteps.writeDataFrame","pkg":"com.acxiom.pipeline.steps"}},{"id":"5874ab64-13c7-404c-8a4f-67ff3b0bc7cf","displayName":"Drop Catalog Object","description":"This step will drop an object from the meta store","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the object to drop"},{"type":"text","name":"objectType","required":false,"defaultValue":"TABLE","parameterType":"String","description":"Type of object to drop"},{"type":"boolean","name":"ifExists","required":false,"defaultValue":"false","parameterType":"Boolean","description":"Flag to control whether existence is checked"},{"type":"boolean","name":"cascade","required":false,"defaultValue":"false","parameterType":"Boolean","description":"Flag to control whether this deletion should cascade"}],"engineMeta":{"spark":"CatalogSteps.drop","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"17be71f9-1492-4404-a355-1cc973694cad","displayName":"Database Exists","description":"Check spark catalog for a database with the given name.","type":"branch","category":"Decision","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the database"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"CatalogSteps.databaseExists","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}}},{"id":"95181811-d83e-4136-bedb-2cba1de90301","displayName":"Table Exists","description":"Check spark catalog for a table with the given name.","type":"branch","category":"Decision","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the table"},{"type":"text","name":"database","required":false,"parameterType":"String","description":"Name of the database"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"CatalogSteps.tableExists","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}}},{"id":"f4adfe70-2ae3-4b8d-85d1-f53e91c8dfad","displayName":"Set Current Database","description":"Set the current default database for the spark session.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the database"}],"engineMeta":{"spark":"CatalogSteps.setCurrentDatabase","pkg":"com.acxiom.pipeline.steps"}},{"id":"663f8c93-0a42-4c43-8263-33f89c498760","displayName":"Create Table","description":"Create a table in the meta store.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the table"},{"type":"text","name":"externalPath","required":false,"parameterType":"String","description":"Path of the external table"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"Options containing the format, schema, and settings"}],"engineMeta":{"spark":"CatalogSteps.createTable","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"87db259d-606e-46eb-b723-82923349640f","displayName":"Load DataFrame from HDFS path","description":"This step will read a dataFrame from the given HDFS path","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"path","required":true,"parameterType":"String","description":"The HDFS path to load data into the DataFrame"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"HDFSSteps.readFromPath","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"8daea683-ecde-44ce-988e-41630d251cb8","displayName":"Load DataFrame from HDFS paths","description":"This step will read a dataFrame from the given HDFS paths","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"paths","required":true,"parameterType":"List[String]","description":"The HDFS paths to load data into the DataFrame"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"HDFSSteps.readFromPaths","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"0a296858-e8b7-43dd-9f55-88d00a7cd8fa","displayName":"Write DataFrame to HDFS","description":"This step will write a dataFrame in a given format to HDFS","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to write"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The GCS path to write data"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The optional DataFrame Options"}],"engineMeta":{"spark":"HDFSSteps.writeToPath","pkg":"com.acxiom.pipeline.steps"}},{"id":"e4dad367-a506-5afd-86c0-82c2cf5cd15c","displayName":"Create HDFS FileManager","description":"Simple function to generate the HDFSFileManager for the local HDFS file system","type":"Pipeline","category":"InputOutput","params":[],"engineMeta":{"spark":"HDFSSteps.createFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.HDFSFileManager"}}},{"id":"a7e17c9d-6956-4be0-a602-5b5db4d1c08b","displayName":"Scala script Step","description":"Executes a script and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"scala","parameterType":"String","description":"A scala script to execute"}],"engineMeta":{"spark":"ScalaSteps.processScript","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}}},{"id":"8bf8cef6-cf32-4d85-99f4-e4687a142f84","displayName":"Scala script Step with additional object provided","description":"Executes a script with the provided object and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"scala","parameterType":"String","description":"A scala script to execute"},{"type":"text","name":"value","required":true,"parameterType":"Any","description":"A value to pass to the script"},{"type":"text","name":"type","required":false,"parameterType":"String","description":"The type of the value to pass to the script"}],"engineMeta":{"spark":"ScalaSteps.processScriptWithValue","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}}},{"id":"3ab721e8-0075-4418-aef1-26abdf3041be","displayName":"Scala script Step with additional objects provided","description":"Executes a script with the provided object and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"scala","parameterType":"String","description":"A scala script to execute"},{"type":"object","name":"values","required":true,"parameterType":"Map[String,Any]","description":"Map of name/value pairs that will be bound to the script"},{"type":"object","name":"types","required":false,"parameterType":"Map[String,String]","description":"Map of type overrides for the values provided"},{"type":"boolean","name":"unwrapOptions","required":false,"parameterType":"Boolean","description":"Flag to toggle option unwrapping behavior"}],"engineMeta":{"spark":"ScalaSteps.processScriptWithValues","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}}},{"id":"6e42b0c3-340e-4848-864c-e1b5c57faa4f","displayName":"Join DataFrames","description":"Join two dataFrames together.","type":"Pipeline","category":"Data","params":[{"type":"text","name":"left","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"Left side of the join"},{"type":"text","name":"right","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"Right side of the join"},{"type":"text","name":"expression","required":false,"parameterType":"String","description":"Join expression. Optional for cross joins"},{"type":"text","name":"leftAlias","required":false,"defaultValue":"left","parameterType":"String","description":"Left side alias"},{"type":"text","name":"rightAlias","required":false,"defaultValue":"right","parameterType":"String","description":"Right side alias"},{"type":"text","name":"joinType","required":false,"defaultValue":"inner","parameterType":"String","description":"Type of join to perform"}],"engineMeta":{"spark":"DataSteps.join","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"823eeb28-ec81-4da6-83f2-24a1e580b0e5","displayName":"Group By","description":"Group by a list of grouping expressions and a list of aggregates.","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to group"},{"type":"text","name":"groupings","required":true,"parameterType":"List[String]","description":"List of expressions to group by"},{"type":"text","name":"aggregations","required":true,"parameterType":"List[String]","description":"List of aggregations to apply"}],"engineMeta":{"spark":"DataSteps.groupBy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"d322769c-18a0-49c2-9875-41446892e733","displayName":"Union","description":"Union two DataFrames together.","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The initial DataFrame"},{"type":"text","name":"append","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The dataFrame to append"},{"type":"boolean","name":"distinct","required":false,"defaultValue":"true","parameterType":"Boolean","description":"Flag to control distinct behavior"}],"engineMeta":{"spark":"DataSteps.union","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}}},{"id":"80583aa9-41b7-4906-8357-cc2d3670d970","displayName":"Add a Column with a Static Value to All Rows in a DataFrame (metalus-common)","description":"This step will add a column with a static value to all rows in the provided data frame","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The data frame to add the column"},{"type":"text","name":"columnName","required":true,"parameterType":"String","description":"The name to provide the id column"},{"type":"text","name":"columnValue","required":true,"parameterType":"Any","description":"The name of the new column"},{"type":"boolean","name":"standardizeColumnName","required":false,"defaultValue":"true","parameterType":"Boolean","description":"The value to add"}],"engineMeta":{"spark":"DataSteps.addStaticColumnToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"e625eed6-51f0-44e7-870b-91c960cdc93d","displayName":"Adds a Unique Identifier to a DataFrame (metalus-common)","description":"This step will add a new unique identifier to an existing data frame using the monotonically_increasing_id method","type":"Pipeline","category":"Data","params":[{"type":"text","name":"idColumnName","required":true,"parameterType":"String","description":"The name to provide the id column"},{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The data frame to add the column"}],"engineMeta":{"spark":"DataSteps.addUniqueIdToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"fa0fcabb-d000-4a5e-9144-692bca618ddb","displayName":"Filter a DataFrame","description":"This step will filter a DataFrame based on the where expression provided","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to filter"},{"type":"text","name":"expression","required":true,"parameterType":"String","description":"The expression to apply to the DataFrame to filter rows"}],"engineMeta":{"spark":"DataSteps.applyFilter","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}}},{"id":"5d0d7c5c-c287-4565-80b2-2b1a847b18c6","displayName":"Get DataFrame Count","description":"Get a count of records in a DataFrame.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to count"}],"engineMeta":{"spark":"DataSteps.getDataFrameCount","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Long"}}},{"id":"252b6086-da45-4042-a9a8-31ebf57948af","displayName":"Drop Duplicate Records","description":"Drop duplicate records from a DataFrame","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to drop duplicate records from"},{"type":"text","name":"columnNames","required":true,"parameterType":"List[String]","description":"Columns to use for determining distinct values to drop"}],"engineMeta":{"spark":"DataSteps.dropDuplicateRecords","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}}},{"id":"d5ac88a2-caa2-473c-a9f7-ffb0269880b2","displayName":"Rename Column","description":"Rename a column on a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to change"},{"type":"text","name":"oldColumnName","required":true,"parameterType":"String","description":"The name of the column you want to change"},{"type":"text","name":"newColumnName","required":true,"parameterType":"String","description":"The new name to give the column"}],"engineMeta":{"spark":"DataSteps.renameColumn","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"a2f3e151-cb81-4c69-8475-c1a287bbb4cb","displayName":"Convert CSV String Dataset to DataFrame","description":"This step will convert the provided CSV string Dataset into a DataFrame that can be passed to other steps","type":"Pipeline","category":"CSV","params":[{"type":"text","name":"dataset","required":true,"parameterType":"org.apache.spark.sql.Dataset[String]","description":"The dataset containing CSV strings"},{"type":"object","name":"dataFrameReaderOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The CSV parsing options"}],"engineMeta":{"spark":"CSVSteps.csvDatasetToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"d25209c1-53f6-49ad-a402-257ae756ac2a","displayName":"Convert CSV String to DataFrame","description":"This step will convert the provided CSV string into a DataFrame that can be passed to other steps","type":"Pipeline","category":"CSV","params":[{"type":"text","name":"csvString","required":true,"parameterType":"String","description":"The csv string to convert to a DataFrame"},{"type":"text","name":"delimiter","required":false,"defaultValue":",","parameterType":"String","description":"The field delimiter"},{"type":"text","name":"recordDelimiter","required":false,"defaultValue":"\\n","parameterType":"String","description":"The record delimiter"},{"type":"boolean","name":"header","required":false,"defaultValue":"false","parameterType":"Boolean","description":"Build header from the first row"}],"engineMeta":{"spark":"CSVSteps.csvStringToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"15889487-fd1c-4c44-b8eb-973c12f91fae","displayName":"Creates an HttpRestClient","description":"This step will build an HttpRestClient using a host url and optional authorization object","type":"Pipeline","category":"API","params":[{"type":"text","name":"hostUrl","required":true,"parameterType":"String","description":"The URL to connect including port"},{"type":"text","name":"authorization","required":false,"parameterType":"com.acxiom.pipeline.api.Authorization","description":"The optional authorization class to use when making connections"},{"type":"boolean","name":"allowSelfSignedCertificates","required":false,"parameterType":"Boolean","description":"Flag to allow using self signed certificates for http calls"}],"engineMeta":{"spark":"ApiSteps.createHttpRestClient","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.api.HttpRestClient"}}},{"id":"fcfd4b91-9a9c-438c-8afa-9f14c1e52a82","displayName":"Creates an HttpRestClient from protocol, host and port","description":"This step will build an HttpRestClient using url parts and optional authorization object","type":"Pipeline","category":"API","params":[{"type":"text","name":"protocol","required":true,"parameterType":"String","description":"The protocol to use when constructing the URL"},{"type":"text","name":"host","required":true,"parameterType":"String","description":"The host name to use when constructing the URL"},{"type":"text","name":"port","required":true,"parameterType":"Int","description":"The port to use when constructing the URL"},{"type":"text","name":"authorization","required":false,"parameterType":"com.acxiom.pipeline.api.Authorization","description":"The optional authorization class to use when making connections"}],"engineMeta":{"spark":"ApiSteps.createHttpRestClientFromParameters","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.api.HttpRestClient"}}},{"id":"b59f0486-78aa-4bd4-baf5-5c7d7c648ff0","displayName":"Check Path Exists","description":"Checks the path to determine whether it exists or not.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to verify"}],"engineMeta":{"spark":"ApiSteps.exists","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}}},{"id":"7521ac47-84ec-4e50-b087-b9de4bf6d514","displayName":"Get the last modified date","description":"Gets the last modified date for the provided path","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource to get the last modified date"}],"engineMeta":{"spark":"ApiSteps.getLastModifiedDate","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.util.Date"}}},{"id":"fff7f7b6-5d9a-40b3-8add-6432552920a8","displayName":"Get Path Content Length","description":"Get the size of the content at the given path.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource to get the content length"}],"engineMeta":{"spark":"ApiSteps.getContentLength","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Long"}}},{"id":"dd351d47-125d-47fa-bafd-203bebad82eb","displayName":"Get Path Headers","description":"Get the headers for the content at the given path.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to get the headers"}],"engineMeta":{"spark":"ApiSteps.getHeaders","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Map[String,List[String]]"}}},{"id":"532f72dd-8443-481d-8406-b74cdc08e342","displayName":"Delete Content","description":"Attempts to delete the provided path..","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to delete"}],"engineMeta":{"spark":"ApiSteps.delete","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}}},{"id":"3b91e6e8-ec18-4468-9089-8474f4b4ba48","displayName":"GET String Content","description":"Retrieves the value at the provided path as a string.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to resource"}],"engineMeta":{"spark":"ApiSteps.getStringContent","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"34c2fc9a-2502-4c79-a0cb-3f866a0a0d6e","displayName":"POST String Content","description":"POSTs the provided string to the provided path using the content type and returns the response as a string.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to post the content"},{"type":"text","name":"content","required":true,"parameterType":"String","description":"The content to post"},{"type":"text","name":"contentType","required":false,"parameterType":"String","description":"The content type being sent to the path"}],"engineMeta":{"spark":"ApiSteps.postStringContent","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"49ae38b3-cb41-4153-9111-aa6aacf6721d","displayName":"PUT String Content","description":"PUTs the provided string to the provided path using the content type and returns the response as a string.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to post the content"},{"type":"text","name":"content","required":true,"parameterType":"String","description":"The content to put"},{"type":"text","name":"contentType","required":false,"parameterType":"String","description":"The content type being sent to the path"}],"engineMeta":{"spark":"ApiSteps.putStringContent","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"99b20c23-722f-4862-9f47-bc9f72440ae6","displayName":"GET Input Stream","description":"Creates a buffered input stream for the provided path","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource"},{"type":"text","name":"bufferSize","required":false,"parameterType":"Int","description":"The size of buffer to use with the stream"}],"engineMeta":{"spark":"ApiSteps.getInputStream","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.io.InputStream"}}},{"id":"f4120b1c-91df-452f-9589-b77f8555ba44","displayName":"GET Output Stream","description":"Creates a buffered output stream for the provided path.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource"},{"type":"text","name":"bufferSize","required":false,"parameterType":"Int","description":"The size of buffer to use with the stream"}],"engineMeta":{"spark":"ApiSteps.getOutputStream","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.io.OutputStream"}}},{"id":"cdb332e3-9ea4-4c96-8b29-c1d74287656c","displayName":"Load table as DataFrame using JDBCOptions","description":"This step will load a table from the provided JDBCOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"jdbcOptions","required":true,"parameterType":"org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions","description":"The options to use when loading the DataFrame"}],"engineMeta":{"spark":"JDBCSteps.readWithJDBCOptions","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"72dbbfc8-bd1d-4ce4-ab35-28fa8385ea54","displayName":"Load table as DataFrame using StepOptions","description":"This step will load a table from the provided JDBCDataFrameReaderOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"object","name":"jDBCStepsOptions","required":true,"className":"com.acxiom.pipeline.steps.JDBCDataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.JDBCDataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"JDBCSteps.readWithStepOptions","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"dcc57409-eb91-48c0-975b-ca109ba30195","displayName":"Load table as DataFrame","description":"This step will load a table from the provided jdbc information","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"url","required":true,"parameterType":"String","description":"A valid jdbc url"},{"type":"text","name":"table","required":true,"parameterType":"String","description":"A table name or subquery"},{"type":"text","name":"predicates","required":false,"parameterType":"List[String]","description":"Optional predicates used for partitioning"},{"type":"text","name":"connectionProperties","required":false,"parameterType":"Map[String,String]","description":"Optional properties for the jdbc connection"}],"engineMeta":{"spark":"JDBCSteps.readWithProperties","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"c9fddf52-34b1-4216-a049-10c33ccd24ab","displayName":"Write DataFrame to table using JDBCOptions","description":"This step will write a DataFrame as a table using JDBCOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to be written"},{"type":"text","name":"jdbcOptions","required":true,"parameterType":"org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions","description":"Options for configuring the JDBC connection"},{"type":"text","name":"saveMode","required":false,"parameterType":"String","description":"The value for the mode option. Defaulted to Overwrite"}],"engineMeta":{"spark":"JDBCSteps.writeWithJDBCOptions","pkg":"com.acxiom.pipeline.steps"}},{"id":"77ffcd02-fbd0-4f79-9b35-ac9dc5fb7190","displayName":"Write DataFrame to table","description":"This step will write a DataFrame to a table using the provided properties","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to be written"},{"type":"text","name":"url","required":true,"parameterType":"String","description":"A valid jdbc url"},{"type":"text","name":"table","required":true,"parameterType":"String","description":"A table name or subquery"},{"type":"text","name":"connectionProperties","required":false,"parameterType":"Map[String,String]","description":"Optional properties for the jdbc connection"},{"type":"text","name":"saveMode","required":false,"parameterType":"String","description":"The value for the mode option. Defaulted to Overwrite"}],"engineMeta":{"spark":"JDBCSteps.writeWithProperties","pkg":"com.acxiom.pipeline.steps"}},{"id":"3d6b77a1-52c2-49ba-99a0-7ec773dac696","displayName":"Write DataFrame to JDBC table","description":"This step will write a DataFrame to a table using the provided JDBCDataFrameWriterOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to be written"},{"type":"object","name":"jDBCStepsOptions","required":true,"className":"com.acxiom.pipeline.steps.JDBCDataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.JDBCDataFrameWriterOptions","description":"Options for the JDBC connect and spark DataFrameWriter"}],"engineMeta":{"spark":"JDBCSteps.writeWithStepOptions","pkg":"com.acxiom.pipeline.steps"}},{"id":"713fff3d-d407-4970-89ae-7844e6fc60e3","displayName":"Get JDBC Connection","description":"Get a jdbc connection.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"url","required":true,"parameterType":"String","description":"A valid jdbc url"},{"type":"text","name":"properties","required":false,"parameterType":"Map[String,String]","description":"Optional properties for the jdbc connection"}],"engineMeta":{"spark":"JDBCSteps.getConnection","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.sql.Connection"}}},{"id":"549828be-3d96-4561-bf94-7ad420f9d203","displayName":"Execute Sql","description":"Execute a sql command using jdbc.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"sql","required":true,"parameterType":"String","description":"Sql command to execute"},{"type":"text","name":"connection","required":true,"parameterType":"java.sql.Connection","description":"An open jdbc connection"},{"type":"text","name":"parameters","required":false,"parameterType":"List[Any]","description":"Optional list of bind variables"}],"engineMeta":{"spark":"JDBCSteps.executeSql","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}}},{"id":"9c8957a3-899e-4f32-830e-d120b1917aa1","displayName":"Close JDBC Connection","description":"Close a JDBC Connection.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"connection","required":true,"parameterType":"java.sql.Connection","description":"An open jdbc connection"}],"engineMeta":{"spark":"JDBCSteps.closeConnection","pkg":"com.acxiom.pipeline.steps"}},{"id":"3464dc85-5111-40fc-9bfb-1fd6fc8a2c17","displayName":"Convert JSON String to Map","description":"This step will convert the provided JSON string into a Map that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonString","required":true,"parameterType":"String","description":"The JSON string to convert to a map"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonStringToMap","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Map[String,Any]"}}},{"id":"f4d19691-779b-4962-a52b-ee5d9a99068e","displayName":"Convert JSON Map to JSON String","description":"This step will convert the provided JSON map into a JSON string that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonMap","required":true,"parameterType":"Map[String,Any]","description":"The JSON map to convert to a JSON string"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonMapToString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"1f23eb37-98ee-43c2-ac78-17b04db3cc8d","displayName":"Convert object to JSON String","description":"This step will convert the provided object into a JSON string that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"obj","required":true,"parameterType":"AnyRef","description":"The object to convert to a JSON string"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.objectToJsonString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"880c5151-f7cd-40bb-99f2-06dbb20a6523","displayName":"Convert JSON String to object","description":"This step will convert the provided JSON string into an object that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonString","required":true,"parameterType":"String","description":"The JSON string to convert to an object"},{"type":"text","name":"objectName","required":true,"parameterType":"String","description":"The fully qualified class name of the object"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonStringToObject","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Any"}}},{"id":"68958a29-aab5-4f7e-9ffd-af99c33c512b","displayName":"Convert JSON String to Schema","description":"This step will convert the provided JSON string into a Schema that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"schema","required":true,"parameterType":"String","description":"The JSON string to convert to a Schema"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonStringToSchema","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.Schema"}}},{"id":"cf4e9e6c-98d6-4a14-ae74-52322782c504","displayName":"Convert JSON String to DataFrame","description":"This step will convert the provided JSON string into a DataFrame that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonString","required":true,"parameterType":"String","description":"The JSON string to convert to a DataFrame"}],"engineMeta":{"spark":"JSONSteps.jsonStringToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"d5cd835e-5e8f-49c0-9706-746d5a4d7b3a","displayName":"Convert JSON String Dataset to DataFrame","description":"This step will convert the provided JSON string Dataset into a DataFrame that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"dataset","required":true,"parameterType":"org.apache.spark.sql.Dataset[String]","description":"The dataset containing JSON strings"},{"type":"object","name":"dataFrameReaderOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The JSON parsing options"}],"engineMeta":{"spark":"JSONSteps.jsonDatasetToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"f3891201-5138-4cab-aebc-bcc319228543","displayName":"Build JSON4S Formats","description":"This step will build a json4s Formats object that can be used to override the default","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"customSerializers","required":false,"parameterType":"List[com.acxiom.pipeline.applications.ClassInfo]","description":"List of custom serializer classes"},{"type":"text","name":"enumIdSerializers","required":false,"parameterType":"List[com.acxiom.pipeline.applications.ClassInfo]","description":"List of Enumeration classes to serialize by id"},{"type":"text","name":"enumNameSerializers","required":false,"parameterType":"List[com.acxiom.pipeline.applications.ClassInfo]","description":"List of Enumeration classes to serialize by name"}],"engineMeta":{"spark":"JSONSteps.buildJsonFormats","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.json4s.Formats"}}},{"id":"b5485d97-d4e8-41a6-8af7-9ce79a435140","displayName":"To String","description":"Returns the result of the toString method, can unwrap options","type":"Pipeline","category":"String","params":[{"type":"text","name":"value","required":true,"parameterType":"Any","description":"The value to convert"},{"type":"boolean","name":"unwrapOption","required":false,"parameterType":"Boolean","description":"Boolean indicating whether to unwrap the value from an Option prior to calling toString"}],"engineMeta":{"spark":"StringSteps.toString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"78e817ec-2bf2-4cbe-acba-e5bc9bdcffc5","displayName":"List To String","description":"Returns the result of the mkString method","type":"Pipeline","category":"String","params":[{"type":"text","name":"list","required":true,"parameterType":"List[Any]","description":"The list to convert"},{"type":"text","name":"separator","required":false,"parameterType":"String","description":"Separator character to use when making the string"},{"type":"boolean","name":"unwrapOptions","required":false,"parameterType":"Boolean","description":"Boolean indicating whether to unwrap each value from an Option"}],"engineMeta":{"spark":"StringSteps.listToString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"fcd6b5fe-08ed-4cfd-acfe-eb676d7f4ecd","displayName":"To Lowercase","description":"Returns a lowercase string","type":"Pipeline","category":"String","params":[{"type":"text","name":"value","required":true,"parameterType":"String","description":"The value to lowercase"}],"engineMeta":{"spark":"StringSteps.toLowerCase","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"2f31ebf1-4ae2-4e04-9b29-4802cac8a198","displayName":"To Uppercase","description":"Returns an uppercase string","type":"Pipeline","category":"String","params":[{"type":"text","name":"value","required":true,"parameterType":"String","description":"The value to uppercase"}],"engineMeta":{"spark":"StringSteps.toUpperCase","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"96b7b521-5304-4e63-8435-63d84a358368","displayName":"String Split","description":"Returns a list of strings split off of the given string","type":"Pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to split"},{"type":"text","name":"regex","required":true,"parameterType":"String","description":"Regex to use when splitting the string"},{"type":"integer","name":"limit","required":false,"parameterType":"Int","description":"Max number elements to return in the list"}],"engineMeta":{"spark":"StringSteps.stringSplit","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"List[String]"}}},{"id":"f75abedd-4aee-4979-8d56-ea7b0c1a86e1","displayName":"Substring","description":"Returns a substring","type":"Pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to parse"},{"type":"text","name":"begin","required":true,"parameterType":"Int","description":"The beginning index"},{"type":"integer","name":"end","required":false,"parameterType":"Int","description":"The end index"}],"engineMeta":{"spark":"StringSteps.substring","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"3fabf9ec-5383-4eb3-81af-6092ab7c370d","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","category":"Decision","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"parameterType":"String","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"parameterType":"Boolean","description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}}},{"id":"ff0562f5-2917-406d-aa78-c5d49ba6b99f","displayName":"String Matches","description":"Return whether string matches a given regex","type":"branch","category":"Decision","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to match"},{"type":"text","name":"regex","required":true,"parameterType":"String","description":"Regex to use for the match"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"StringSteps.stringMatches","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}}},{"id":"416baf4e-a1dd-49fc-83a9-0f41b77e57b7","displayName":"String Replace All","description":"Perform a literal or regex replacement on a string","type":"pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to modify"},{"type":"text","name":"matchString","required":true,"parameterType":"String","description":"The string to match"},{"type":"text","name":"replacement","required":false,"parameterType":"String","description":"The replacement string"},{"type":"boolean","name":"literal","required":false,"parameterType":"Boolean","description":"Perform \\'literal\\' match replacement"}],"engineMeta":{"spark":"StringSteps.stringReplaceAll","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"95438b82-8d50-41da-8094-c92449b9e7df","displayName":"String Replace First","description":"Perform a literal or regex replacement on the first occurrence in a string","type":"pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to modify"},{"type":"text","name":"matchString","required":true,"parameterType":"String","description":"The string to match"},{"type":"text","name":"replacement","required":false,"parameterType":"String","description":"The replacement string"},{"type":"boolean","name":"literal","required":false,"parameterType":"Boolean","description":"Perform \\'literal\\' match replacement"}],"engineMeta":{"spark":"StringSteps.stringReplaceFirst","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"86c84fa3-ad45-4a49-ac05-92385b8e9572","displayName":"Get Credential","description":"This step provides access to credentials through the CredentialProvider","type":"Pipeline","category":"Credentials","params":[{"type":"text","name":"credentialName","required":true,"parameterType":"String","description":"The dataset containing CSV strings"}],"engineMeta":{"spark":"CredentialSteps.getCredential","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.Credential"}}},{"id":"219c787a-f502-4efc-b15d-5beeff661fc0","displayName":"Map a DataFrame to an existing DataFrame","description":"This step maps a new DataFrame to an existing DataFrame to make them compatible","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"inputDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame that needs to be modified"},{"type":"text","name":"destinationDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame that the new data needs to map to"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations","description":"The object with transform, alias, and filter logic details"},{"type":"boolean","name":"addNewColumns","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.mapToDestinationDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"8f9c08ea-4882-4265-bac7-2da3e942758f","displayName":"Map a DataFrame to a pre-defined Schema","description":"This step maps a new DataFrame to a pre-defined spark schema","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"inputDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame that needs to be modified"},{"type":"object","name":"destinationSchema","required":true,"className":"com.acxiom.pipeline.steps.Schema","parameterType":"com.acxiom.pipeline.steps.Schema","description":"The schema that the new data should map to"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations","description":"The object with transform, alias, and filter logic details"},{"type":"boolean","name":"addNewColumns","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.mapDataFrameToSchema","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"3ee74590-9131-43e1-8ee8-ad320482a592","displayName":"Merge a DataFrame to an existing DataFrame","description":"This step merges two DataFrames to create a single DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"inputDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The first DataFrame"},{"type":"text","name":"destinationDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The second DataFrame used as the driver"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations","description":"The object with transform, alias, and filter logic details"},{"type":"boolean","name":"addNewColumns","required":false,"parameterType":"Boolean"},{"type":"boolean","name":"distinct","required":false,"defaultValue":"true","parameterType":"Boolean","description":"Flag to determine whether a distinct union should be performed"}],"engineMeta":{"spark":"TransformationSteps.mergeDataFrames","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"ac3dafe4-e6ee-45c9-8fc6-fa7f918cf4f2","displayName":"Modify or Create Columns using Transforms Provided","description":"This step transforms existing columns and/or adds new columns to an existing dataframe using expressions provided","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The input DataFrame"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations"}],"engineMeta":{"spark":"TransformationSteps.applyTransforms","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"3e2da5a8-387d-49b1-be22-c03764fb0fde","displayName":"Select Expressions","description":"Select each provided expresion from a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to select from"},{"type":"text","name":"expressions","required":true,"parameterType":"List[String]","description":"List of expressions to select"}],"engineMeta":{"spark":"TransformationSteps.selectExpressions","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"1e0a234a-8ae5-4627-be6d-3052b33d9014","displayName":"Add Column","description":"Add a new column to a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to add to"},{"type":"text","name":"columnName","required":true,"parameterType":"String","description":"The name of the new column"},{"type":"text","name":"expression","required":true,"parameterType":"String","description":"The expression used for the column"},{"type":"boolean","name":"standardizeColumnName","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.addColumn","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"08c9c5a9-a10d-477e-a702-19bd24889d1e","displayName":"Add Columns","description":"Add multiple new columns to a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to add to"},{"type":"text","name":"columns","required":true,"parameterType":"Map[String,String]","description":"A map of column names and expressions"},{"type":"boolean","name":"standardizeColumnNames","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.addColumns","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"42c328ac-a6bd-49ca-b597-b706956d294c","displayName":"Flatten a DataFrame","description":"This step will flatten all nested fields contained in a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to flatten"},{"type":"text","name":"separator","required":false,"defaultValue":"_","parameterType":"String","description":"Separator to place between nested field names"},{"type":"text","name":"fieldList","required":false,"parameterType":"List[String]","description":"List of fields to flatten. Will flatten all fields if left empty"},{"type":"integer","name":"depth","required":false,"parameterType":"Int","description":"How deep should we traverse when flattening."}],"engineMeta":{"spark":"TransformationSteps.flattenDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[_]"}}},{"id":"a981080d-714c-4d36-8b09-d95842ec5655","displayName":"Standardize Column Names on a DataFrame","description":"This step will standardize columns names on existing DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.DataFrame"}],"engineMeta":{"spark":"TransformationSteps.standardizeColumnNames","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"541c4f7d-3524-4d53-bbd9-9f2cfd9d1bd1","displayName":"Save a Dataframe to a TempView","description":"This step stores an existing dataframe to a TempView to be used in future queries in the session","type":"Pipeline","category":"Query","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The dataframe to store"},{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to create (optional, random name will be created if not provided)"}],"engineMeta":{"spark":"QuerySteps.dataFrameToTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"71b71ef3-eaa7-4a1f-b3f3-603a1a54846d","displayName":"Create a TempView from a Query","description":"This step runs a SQL statement against existing TempViews from this session and returns a new TempView","type":"Pipeline","category":"Query","params":[{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"},{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to create (optional, random name will be created if not provided)"}],"engineMeta":{"spark":"QuerySteps.queryToTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"61378ed6-8a4f-4e6d-9c92-6863c9503a54","displayName":"Create a DataFrame from a Query","description":"This step runs a SQL statement against existing TempViews from this session and returns a new DataFrame","type":"Pipeline","category":"Query","params":[{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"}],"engineMeta":{"spark":"QuerySteps.queryToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"57b0e491-e09b-4428-aab2-cebe1f217eda","displayName":"Create a DataFrame from an Existing TempView","description":"This step pulls an existing TempView from this session into a new DataFrame","type":"Pipeline","category":"Query","params":[{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to use"}],"engineMeta":{"spark":"QuerySteps.tempViewToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"648f27aa-6e3b-44ed-a093-bc284783731b","displayName":"Create a TempView from a DataFrame Query","description":"This step runs a SQL statement against an existing DataFrame from this session and returns a new TempView","type":"Pipeline","category":"Query","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The dataframe to query"},{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"},{"type":"text","name":"inputViewName","required":true,"parameterType":"String","description":"The name to use when creating the view representing the input dataframe (same name used in query)"},{"type":"text","name":"outputViewName","required":false,"parameterType":"String","description":"The name of the view to create (optional, random name will be created if not provided)"}],"engineMeta":{"spark":"QuerySteps.dataFrameQueryToTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}}},{"id":"dfb8a387-6245-4b1c-ae6c-94067eb83962","displayName":"Create a DataFrame from a DataFrame Query","description":"This step runs a SQL statement against an existing DataFrame from this session and returns a new DataFrame","type":"Pipeline","category":"Query","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The dataframe to query"},{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"},{"type":"text","name":"inputViewName","required":true,"parameterType":"String","description":"The name to use when creating the view representing the input dataframe (same name used in query)"}],"engineMeta":{"spark":"QuerySteps.dataFrameQueryToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"c88de095-14e0-4c67-8537-0325127e2bd2","displayName":"Cache an exising TempView","description":"This step will cache an existing TempView","type":"Pipeline","category":"Query","params":[{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to cache"}],"engineMeta":{"spark":"QuerySteps.cacheTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"0342654c-2722-56fe-ba22-e342169545af","displayName":"Copy source contents to destination","description":"Copy the contents of the source path to the destination path. This function will call connect on both FileManagers.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}}},{"id":"c40169a3-1e77-51ab-9e0a-3f24fb98beef","displayName":"Copy source contents to destination with buffering","description":"Copy the contents of the source path to the destination path using buffer sizes. This function will call connect on both FileManagers.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to"},{"type":"text","name":"inputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for reading data during copy"},{"type":"text","name":"outputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for writing data during copy"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}}},{"id":"f5a24db0-e91b-5c88-8e67-ab5cff09c883","displayName":"Buffered file copy","description":"Copy the contents of the source path to the destination path using full buffer sizes. This function will call connect on both FileManagers.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to"},{"type":"text","name":"inputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for reading data during copy"},{"type":"text","name":"outputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for writing data during copy"},{"type":"text","name":"copyBufferSize","required":true,"parameterType":"Int","description":"The intermediate buffer size to use during copy"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}}},{"id":"3d1e8519-690c-55f0-bd05-1e7b97fb6633","displayName":"Disconnect a FileManager","description":"Disconnects a FileManager from the underlying file system","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The file manager to disconnect"}],"engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"}},{"id":"9d467cb0-8b3d-40a0-9ccd-9cf8c5b6cb38","displayName":"Create SFTP FileManager","description":"Simple function to generate the SFTPFileManager for the remote SFTP file system","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"hostName","required":true,"parameterType":"String","description":"The name of the host to connect"},{"type":"text","name":"username","required":false,"parameterType":"String","description":"The username used for connection"},{"type":"text","name":"password","required":false,"parameterType":"String","description":"The password used for connection"},{"type":"integer","name":"port","required":false,"parameterType":"Int","description":"The optional port if other than 22"},{"type":"boolean","name":"strictHostChecking","required":false,"parameterType":"Boolean","description":"Option to automatically add keys to the known_hosts file. Default is false."}],"engineMeta":{"spark":"SFTPSteps.createFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.SFTPFileManager"}}},{"id":"22fcc0e7-0190-461c-a999-9116b77d5919","displayName":"Build a DataFrameReader Object","description":"This step will build a DataFrameReader object that can be used to read a file into a dataframe","type":"Pipeline","category":"InputOutput","params":[{"type":"object","name":"dataFrameReaderOptions","required":true,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"DataFrameSteps.getDataFrameReader","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrameReader"}}},{"id":"66a451c8-ffbd-4481-9c37-71777c3a240f","displayName":"Load Using DataFrameReader","description":"This step will load a DataFrame given a dataFrameReader.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrameReader","required":true,"parameterType":"org.apache.spark.sql.DataFrameReader","description":"The DataFrameReader to use when creating the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.load","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"d7cf27e6-9ca5-4a73-a1b3-d007499f235f","displayName":"Load DataFrame","description":"This step will load a DataFrame given a DataFrameReaderOptions object.","type":"Pipeline","category":"InputOutput","params":[{"type":"object","name":"dataFrameReaderOptions","required":true,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The DataFrameReaderOptions to use when creating the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.loadDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}}},{"id":"8a00dcf8-e6a9-4833-871e-c1f3397ab378","displayName":"Build a DataFrameWriter Object","description":"This step will build a DataFrameWriter object that can be used to write a file into a dataframe","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to use when creating the DataFrameWriter"},{"type":"object","name":"options","required":true,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The DataFrameWriterOptions to use when writing the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.getDataFrameWriter","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrameWriter[T]"}}},{"id":"9aa6ae9f-cbeb-4b36-ba6a-02eee0a46558","displayName":"Save Using DataFrameWriter","description":"This step will save a DataFrame given a dataFrameWriter[Row].","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrameWriter","required":true,"parameterType":"org.apache.spark.sql.DataFrameWriter[_]","description":"The DataFrameWriter to use when saving"}],"engineMeta":{"spark":"DataFrameSteps.save","pkg":"com.acxiom.pipeline.steps"}},{"id":"e5ac3671-ee10-4d4e-8206-fec7effdf7b9","displayName":"Save DataFrame","description":"This step will save a DataFrame given a DataFrameWriterOptions object.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to save"},{"type":"object","name":"dataFrameWriterOptions","required":true,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The DataFrameWriterOptions to use for saving"}],"engineMeta":{"spark":"DataFrameSteps.saveDataFrame","pkg":"com.acxiom.pipeline.steps"}},{"id":"fa05a970-476d-4617-be4d-950cfa65f2f8","displayName":"Persist DataFrame","description":"Persist a DataFrame to provided storage level.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to persist"},{"type":"text","name":"storageLevel","required":false,"parameterType":"String","description":"The optional storage mechanism to use when persisting the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.persistDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}}},{"id":"e6fe074e-a1fa-476f-9569-d37295062186","displayName":"Unpersist DataFrame","description":"Unpersist a DataFrame.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to unpersist"},{"type":"boolean","name":"blocking","required":false,"parameterType":"Boolean","description":"Optional flag to indicate whether to block while unpersisting"}],"engineMeta":{"spark":"DataFrameSteps.unpersistDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}}},{"id":"71323226-bcfd-4fa1-bf9e-24e455e41144","displayName":"RepartitionDataFrame","description":"Repartition a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to repartition"},{"type":"text","name":"partitions","required":true,"parameterType":"Int","description":"The number of partitions to use"},{"type":"boolean","name":"rangePartition","required":false,"parameterType":"Boolean","description":"Flag indicating whether to repartition by range. This takes precedent over the shuffle flag"},{"type":"boolean","name":"shuffle","required":false,"parameterType":"Boolean","description":"Flag indicating whether to perform a normal partition"},{"type":"text","name":"partitionExpressions","required":false,"parameterType":"List[String]","description":"The partition expressions to use"}],"engineMeta":{"spark":"DataFrameSteps.repartitionDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}}},{"id":"5e0358a0-d567-5508-af61-c35a69286e4e","displayName":"Javascript Step","description":"Executes a script and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"javascript","parameterType":"String","description":"Javascript to execute"}],"engineMeta":{"spark":"JavascriptSteps.processScript","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}}},{"id":"570c9a80-8bd1-5f0c-9ae0-605921fe51e2","displayName":"Javascript Step with single object provided","description":"Executes a script with single object provided and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"javascript","parameterType":"String","description":"Javascript script to execute"},{"type":"text","name":"value","required":true,"parameterType":"Any","description":"Value to bind to the script"}],"engineMeta":{"spark":"JavascriptSteps.processScriptWithValue","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}}},{"id":"f92d4816-3c62-4c29-b420-f00994bfcd86","displayName":"Javascript Step with map of objects provided","description":"Executes a script with map of objects provided and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"javascript","parameterType":"String"},{"type":"text","name":"values","required":true,"parameterType":"Map[String,Any]","description":"Map of name/value pairs to bind to the script"},{"type":"boolean","name":"unwrapOptions","required":false,"parameterType":"Boolean","description":"Flag to control option unwrapping behavior"}],"engineMeta":{"spark":"JavascriptSteps.processScriptWithValues","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}}}],"pkgObjs":[{"id":"com.acxiom.pipeline.steps.JDBCDataFrameReaderOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"JDBC Data Frame Reader Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"url\":{\"type\":\"string\"},\"table\":{\"type\":\"string\"},\"predicates\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"readerOptions\":{\"$ref\":\"#/definitions/DataFrameReaderOptions\"}},\"definitions\":{\"DataFrameReaderOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}},\"Schema\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}}},\"Attribute\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"name\":{\"type\":\"string\"},\"dataType\":{\"$ref\":\"#/definitions/AttributeType\"}}},\"AttributeType\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"baseType\":{\"type\":\"string\"},\"valueType\":{\"$ref\":\"#/definitions/AttributeType\"},\"nameType\":{\"$ref\":\"#/definitions/AttributeType\"},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}}}}"},{"id":"com.acxiom.pipeline.steps.DataFrameWriterOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Data Frame Writer Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"saveMode\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"bucketingOptions\":{\"$ref\":\"#/definitions/BucketingOptions\"},\"partitionBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"sortBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}},\"definitions\":{\"BucketingOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"numBuckets\":{\"type\":\"integer\"},\"columns\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}},\"required\":[\"numBuckets\"]}}}","template":"{\"form\":[{\"type\":\"select\",\"key\":\"format\",\"templateOptions\":{\"label\":\"Format\",\"placeholder\":\"\",\"valueProp\":\"value\",\"options\":[{\"value\":\"csv\",\"name\":\"CSV\"},{\"value\":\"json\",\"name\":\"JSON\"},{\"value\":\"parquet\",\"name\":\"Parquet\"},{\"value\":\"orc\",\"name\":\"Orc\"},{\"value\":\"text\",\"name\":\"Text\"}],\"labelProp\":\"name\",\"focus\":false,\"_flatOptions\":true,\"disabled\":false}},{\"key\":\"options.encoding\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Encoding\",\"placeholder\":\"\",\"focus\":false},\"expressionProperties\":{\"templateOptions.disabled\":\"!model.format\"}},{\"key\":\"saveMode\",\"type\":\"select\",\"defaultValue\":false,\"templateOptions\":{\"label\":\"Save Mode\",\"placeholder\":\"\",\"valueProp\":\"value\",\"options\":[{\"value\":\"OVERWRITE\",\"name\":\"Overwrite\"},{\"value\":\"append\",\"name\":\"Append\"},{\"value\":\"ignore\",\"name\":\"Ignore\"},{\"value\":\"error\",\"name\":\"Error\"},{\"value\":\"errorifexists\",\"name\":\"Error If Exists\"}],\"labelProp\":\"name\",\"focus\":false,\"_flatOptions\":true,\"disabled\":false}},{\"key\":\"partitionBy\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Partition By Columns\",\"placeholder\":\"Add column names to use during partitioning\"}},{\"key\":\"sortBy\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Sort By Columns\",\"placeholder\":\"Add column names to use during sorting\"}},{\"key\":\"bucketingOptions\",\"wrappers\":[\"panel\"],\"templateOptions\":{\"label\":\"Bucketing Options\"},\"fieldGroup\":[{\"key\":\"numBuckets\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Number of Buckets\",\"type\":\"number\"}},{\"key\":\"columns\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Bucket Columns\",\"placeholder\":\"Add column names to use during bucketing\"}}]},{\"key\":\"options.sep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.lineSep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Line Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'text' || model.format === 'json' ? false : true\"},{\"key\":\"options.escapeQuotes\",\"hideExpression\":\"model.format === 'csv' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Escape Quotes?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.quote\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Quote\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.escape\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Escape Character\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"}]}"},{"id":"com.acxiom.pipeline.steps.JDBCDataFrameWriterOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"JDBC Data Frame Writer Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"url\":{\"type\":\"string\"},\"table\":{\"type\":\"string\"},\"writerOptions\":{\"$ref\":\"#/definitions/DataFrameWriterOptions\"}},\"definitions\":{\"DataFrameWriterOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"saveMode\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"bucketingOptions\":{\"$ref\":\"#/definitions/BucketingOptions\"},\"partitionBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"sortBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}}},\"BucketingOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"numBuckets\":{\"type\":\"integer\"},\"columns\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}},\"required\":[\"numBuckets\"]}}}"},{"id":"com.acxiom.pipeline.steps.Transformations","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Transformations\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"columnDetails\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/ColumnDetails\"}},\"filter\":{\"type\":\"string\"},\"standardizeColumnNames\":{}},\"definitions\":{\"ColumnDetails\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"outputField\":{\"type\":\"string\"},\"inputAliases\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"expression\":{\"type\":\"string\"}}}}}","template":"{\"form\":[{\"key\":\"filter\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Filter\"}},{\"key\":\"standardizeColumnNames\",\"type\":\"checkbox\",\"defaultValue\":false,\"templateOptions\":{\"floatLabel\":\"always\",\"align\":\"start\",\"label\":\"Standardize Column Names?\",\"hideFieldUnderline\":true,\"color\":\"accent\",\"placeholder\":\"\",\"focus\":false,\"hideLabel\":true,\"disabled\":false,\"indeterminate\":true}},{\"fieldArray\":{\"fieldGroup\":[{\"key\":\"outputField\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Output Field\"}},{\"key\":\"inputAliases\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Input Alias\"}},{\"key\":\"expression\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Expression (Optional)\"}}]},\"key\":\"columnDetails\",\"wrappers\":[\"panel\"],\"type\":\"repeat\",\"templateOptions\":{\"label\":\"Column Details\"}}]}"},{"id":"com.acxiom.pipeline.steps.DataFrameReaderOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Data Frame Reader Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"schema\":{\"$ref\":\"#/definitions/Schema\"}},\"definitions\":{\"Schema\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}}},\"Attribute\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"name\":{\"type\":\"string\"},\"dataType\":{\"$ref\":\"#/definitions/AttributeType\"}}},\"AttributeType\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"baseType\":{\"type\":\"string\"},\"valueType\":{\"$ref\":\"#/definitions/AttributeType\"},\"nameType\":{\"$ref\":\"#/definitions/AttributeType\"},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}}}}","template":"{\"form\":[{\"type\":\"select\",\"key\":\"format\",\"templateOptions\":{\"label\":\"Format\",\"placeholder\":\"\",\"valueProp\":\"value\",\"options\":[{\"value\":\"csv\",\"name\":\"CSV\"},{\"value\":\"json\",\"name\":\"JSON\"},{\"value\":\"parquet\",\"name\":\"Parquet\"},{\"value\":\"orc\",\"name\":\"Orc\"},{\"value\":\"text\",\"name\":\"Text\"}],\"labelProp\":\"name\",\"focus\":false,\"_flatOptions\":true,\"disabled\":false}},{\"key\":\"options.encoding\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Encoding\",\"placeholder\":\"\",\"focus\":false},\"expressionProperties\":{\"templateOptions.disabled\":\"!model.format\"}},{\"key\":\"options.multiLine\",\"hideExpression\":\"model.format === 'csv' || model.format === 'json' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Multiline?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.header\",\"hideExpression\":\"model.format === 'csv' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Skip Header?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.sep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.lineSep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Line Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'text' || model.format === 'json' ? false : true\"},{\"key\":\"options.quote\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Quote\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.escape\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Escape Character\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.primitivesAsString\",\"hideExpression\":\"model.format === 'json' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Primitive As String?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.inferSchema\",\"hideExpression\":\"model.format === 'csv' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Infer Schema?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"expressionProperties\":{\"templateOptions.disabled\":\"model.options.inferSchema || model.format === 'json' ? false : true\"},\"key\":\"options.samplingRatio\",\"hideExpression\":\"model.format === 'csv' || model.format === 'json' ? false : true\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Sampling Ration\",\"type\":\"number\"}}]}"},{"id":"com.acxiom.pipeline.steps.Schema","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Schema\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}},\"definitions\":{\"Attribute\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"name\":{\"type\":\"string\"},\"dataType\":{\"$ref\":\"#/definitions/AttributeType\"}}},\"AttributeType\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"baseType\":{\"type\":\"string\"},\"valueType\":{\"$ref\":\"#/definitions/AttributeType\"},\"nameType\":{\"$ref\":\"#/definitions/AttributeType\"},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}},\"Schema\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}}}}}"}] } \ No newline at end of file +{"pkgs":["com.acxiom.pipeline.steps"],"steps":[{"id":"3806f23b-478c-4054-b6c1-37f11db58d38","displayName":"Read a DataFrame from Table","description":"This step will read a dataFrame in a given format from the meta store","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"table","required":true,"parameterType":"String","description":"The name of the table to read"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The DataFrameReaderOptions to use"}],"engineMeta":{"spark":"CatalogSteps.readDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"e2b4c011-e71b-46f9-a8be-cf937abc2ec4","displayName":"Write DataFrame to Table","description":"This step will write a dataFrame in a given format to the meta store","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to write"},{"type":"text","name":"table","required":true,"parameterType":"String","description":"The name of the table to write to"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The DataFrameWriterOptions to use"}],"engineMeta":{"spark":"CatalogSteps.writeDataFrame","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"5874ab64-13c7-404c-8a4f-67ff3b0bc7cf","displayName":"Drop Catalog Object","description":"This step will drop an object from the meta store","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the object to drop"},{"type":"text","name":"objectType","required":false,"defaultValue":"TABLE","parameterType":"String","description":"Type of object to drop"},{"type":"boolean","name":"ifExists","required":false,"defaultValue":"false","parameterType":"Boolean","description":"Flag to control whether existence is checked"},{"type":"boolean","name":"cascade","required":false,"defaultValue":"false","parameterType":"Boolean","description":"Flag to control whether this deletion should cascade"}],"engineMeta":{"spark":"CatalogSteps.drop","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"17be71f9-1492-4404-a355-1cc973694cad","displayName":"Database Exists","description":"Check spark catalog for a database with the given name.","type":"branch","category":"Decision","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the database"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"CatalogSteps.databaseExists","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"95181811-d83e-4136-bedb-2cba1de90301","displayName":"Table Exists","description":"Check spark catalog for a table with the given name.","type":"branch","category":"Decision","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the table"},{"type":"text","name":"database","required":false,"parameterType":"String","description":"Name of the database"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"CatalogSteps.tableExists","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f4adfe70-2ae3-4b8d-85d1-f53e91c8dfad","displayName":"Set Current Database","description":"Set the current default database for the spark session.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the database"}],"engineMeta":{"spark":"CatalogSteps.setCurrentDatabase","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"663f8c93-0a42-4c43-8263-33f89c498760","displayName":"Create Table","description":"Create a table in the meta store.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the table"},{"type":"text","name":"externalPath","required":false,"parameterType":"String","description":"Path of the external table"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"Options containing the format, schema, and settings"}],"engineMeta":{"spark":"CatalogSteps.createTable","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"836aab38-1140-4606-ab73-5b6744f0e7e7","displayName":"Load","description":"This step will create a DataFrame using the given DataConnector","type":"Pipeline","category":"Connectors","params":[{"type":"text","name":"connector","required":true,"parameterType":"com.acxiom.pipeline.connectors.DataConnector","description":"The data connector to use when writing"},{"type":"text","name":"source","required":false,"parameterType":"String","description":"The source path to load data"},{"type":"object","name":"readOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The optional options to use while reading the data"}],"engineMeta":{"spark":"DataConnectorSteps.loadDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"5608eba7-e9ff-48e6-af77-b5e810b99d89","displayName":"Write","description":"This step will write a DataFrame using the given DataConnector","type":"Pipeline","category":"Connectors","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.DataFrame","description":"The DataFrame to write"},{"type":"text","name":"connector","required":true,"parameterType":"com.acxiom.pipeline.connectors.DataConnector","description":"The data connector to use when writing"},{"type":"text","name":"destination","required":false,"parameterType":"String","description":"The destination path to write data"},{"type":"object","name":"writeOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The optional DataFrame options to use while writing"}],"engineMeta":{"spark":"DataConnectorSteps.writeDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.streaming.StreamingQuery"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"87db259d-606e-46eb-b723-82923349640f","displayName":"Load DataFrame from HDFS path","description":"This step will read a dataFrame from the given HDFS path","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"path","required":true,"parameterType":"String","description":"The HDFS path to load data into the DataFrame"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"HDFSSteps.readFromPath","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"8daea683-ecde-44ce-988e-41630d251cb8","displayName":"Load DataFrame from HDFS paths","description":"This step will read a dataFrame from the given HDFS paths","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"paths","required":true,"parameterType":"List[String]","description":"The HDFS paths to load data into the DataFrame"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"HDFSSteps.readFromPaths","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"0a296858-e8b7-43dd-9f55-88d00a7cd8fa","displayName":"Write DataFrame to HDFS","description":"This step will write a dataFrame in a given format to HDFS","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to write"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The GCS path to write data"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The optional DataFrame Options"}],"engineMeta":{"spark":"HDFSSteps.writeToPath","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"e4dad367-a506-5afd-86c0-82c2cf5cd15c","displayName":"Create HDFS FileManager","description":"Simple function to generate the HDFSFileManager for the local HDFS file system","type":"Pipeline","category":"InputOutput","params":[],"engineMeta":{"spark":"HDFSSteps.createFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.HDFSFileManager"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"a7e17c9d-6956-4be0-a602-5b5db4d1c08b","displayName":"Scala script Step","description":"Executes a script and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"scala","parameterType":"String","description":"A scala script to execute"}],"engineMeta":{"spark":"ScalaSteps.processScript","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"8bf8cef6-cf32-4d85-99f4-e4687a142f84","displayName":"Scala script Step with additional object provided","description":"Executes a script with the provided object and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"scala","parameterType":"String","description":"A scala script to execute"},{"type":"text","name":"value","required":true,"parameterType":"Any","description":"A value to pass to the script"},{"type":"text","name":"type","required":false,"parameterType":"String","description":"The type of the value to pass to the script"}],"engineMeta":{"spark":"ScalaSteps.processScriptWithValue","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3ab721e8-0075-4418-aef1-26abdf3041be","displayName":"Scala script Step with additional objects provided","description":"Executes a script with the provided object and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"scala","parameterType":"String","description":"A scala script to execute"},{"type":"object","name":"values","required":true,"parameterType":"Map[String,Any]","description":"Map of name/value pairs that will be bound to the script"},{"type":"object","name":"types","required":false,"parameterType":"Map[String,String]","description":"Map of type overrides for the values provided"},{"type":"boolean","name":"unwrapOptions","required":false,"parameterType":"Boolean","description":"Flag to toggle option unwrapping behavior"}],"engineMeta":{"spark":"ScalaSteps.processScriptWithValues","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"6e42b0c3-340e-4848-864c-e1b5c57faa4f","displayName":"Join DataFrames","description":"Join two dataFrames together.","type":"Pipeline","category":"Data","params":[{"type":"text","name":"left","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"Left side of the join"},{"type":"text","name":"right","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"Right side of the join"},{"type":"text","name":"expression","required":false,"parameterType":"String","description":"Join expression. Optional for cross joins"},{"type":"text","name":"leftAlias","required":false,"defaultValue":"left","parameterType":"String","description":"Left side alias"},{"type":"text","name":"rightAlias","required":false,"defaultValue":"right","parameterType":"String","description":"Right side alias"},{"type":"text","name":"joinType","required":false,"defaultValue":"inner","parameterType":"String","description":"Type of join to perform"}],"engineMeta":{"spark":"DataSteps.join","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"823eeb28-ec81-4da6-83f2-24a1e580b0e5","displayName":"Group By","description":"Group by a list of grouping expressions and a list of aggregates.","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to group"},{"type":"text","name":"groupings","required":true,"parameterType":"List[String]","description":"List of expressions to group by"},{"type":"text","name":"aggregations","required":true,"parameterType":"List[String]","description":"List of aggregations to apply"}],"engineMeta":{"spark":"DataSteps.groupBy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"d322769c-18a0-49c2-9875-41446892e733","displayName":"Union","description":"Union two DataFrames together.","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The initial DataFrame"},{"type":"text","name":"append","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The dataFrame to append"},{"type":"boolean","name":"distinct","required":false,"defaultValue":"true","parameterType":"Boolean","description":"Flag to control distinct behavior"}],"engineMeta":{"spark":"DataSteps.union","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"80583aa9-41b7-4906-8357-cc2d3670d970","displayName":"Add a Column with a Static Value to All Rows in a DataFrame (metalus-common)","description":"This step will add a column with a static value to all rows in the provided data frame","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The data frame to add the column"},{"type":"text","name":"columnName","required":true,"parameterType":"String","description":"The name to provide the id column"},{"type":"text","name":"columnValue","required":true,"parameterType":"Any","description":"The name of the new column"},{"type":"boolean","name":"standardizeColumnName","required":false,"defaultValue":"true","parameterType":"Boolean","description":"The value to add"}],"engineMeta":{"spark":"DataSteps.addStaticColumnToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"e625eed6-51f0-44e7-870b-91c960cdc93d","displayName":"Adds a Unique Identifier to a DataFrame (metalus-common)","description":"This step will add a new unique identifier to an existing data frame using the monotonically_increasing_id method","type":"Pipeline","category":"Data","params":[{"type":"text","name":"idColumnName","required":true,"parameterType":"String","description":"The name to provide the id column"},{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The data frame to add the column"}],"engineMeta":{"spark":"DataSteps.addUniqueIdToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"fa0fcabb-d000-4a5e-9144-692bca618ddb","displayName":"Filter a DataFrame","description":"This step will filter a DataFrame based on the where expression provided","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to filter"},{"type":"text","name":"expression","required":true,"parameterType":"String","description":"The expression to apply to the DataFrame to filter rows"}],"engineMeta":{"spark":"DataSteps.applyFilter","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"5d0d7c5c-c287-4565-80b2-2b1a847b18c6","displayName":"Get DataFrame Count","description":"Get a count of records in a DataFrame.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to count"}],"engineMeta":{"spark":"DataSteps.getDataFrameCount","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Long"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"252b6086-da45-4042-a9a8-31ebf57948af","displayName":"Drop Duplicate Records","description":"Drop duplicate records from a DataFrame","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to drop duplicate records from"},{"type":"text","name":"columnNames","required":true,"parameterType":"List[String]","description":"Columns to use for determining distinct values to drop"}],"engineMeta":{"spark":"DataSteps.dropDuplicateRecords","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"d5ac88a2-caa2-473c-a9f7-ffb0269880b2","displayName":"Rename Column","description":"Rename a column on a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to change"},{"type":"text","name":"oldColumnName","required":true,"parameterType":"String","description":"The name of the column you want to change"},{"type":"text","name":"newColumnName","required":true,"parameterType":"String","description":"The new name to give the column"}],"engineMeta":{"spark":"DataSteps.renameColumn","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"6ed36f89-35d1-4280-a555-fbcd8dd76bf2","displayName":"Retry (simple)","description":"Makes a decision to retry or stop based on a named counter","type":"branch","category":"RetryLogic","params":[{"type":"text","name":"counterName","required":true,"parameterType":"String","description":"The name of the counter to use for tracking"},{"type":"text","name":"maxRetries","required":true,"parameterType":"Int","description":"The maximum number of retries allowed"},{"type":"result","name":"retry","required":false},{"type":"result","name":"stop","required":false}],"engineMeta":{"spark":"FlowUtilsSteps.simpleRetry","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"a2f3e151-cb81-4c69-8475-c1a287bbb4cb","displayName":"Convert CSV String Dataset to DataFrame","description":"This step will convert the provided CSV string Dataset into a DataFrame that can be passed to other steps","type":"Pipeline","category":"CSV","params":[{"type":"text","name":"dataset","required":true,"parameterType":"org.apache.spark.sql.Dataset[String]","description":"The dataset containing CSV strings"},{"type":"object","name":"dataFrameReaderOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The CSV parsing options"}],"engineMeta":{"spark":"CSVSteps.csvDatasetToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"d25209c1-53f6-49ad-a402-257ae756ac2a","displayName":"Convert CSV String to DataFrame","description":"This step will convert the provided CSV string into a DataFrame that can be passed to other steps","type":"Pipeline","category":"CSV","params":[{"type":"text","name":"csvString","required":true,"parameterType":"String","description":"The csv string to convert to a DataFrame"},{"type":"text","name":"delimiter","required":false,"defaultValue":",","parameterType":"String","description":"The field delimiter"},{"type":"text","name":"recordDelimiter","required":false,"defaultValue":"\\n","parameterType":"String","description":"The record delimiter"},{"type":"boolean","name":"header","required":false,"defaultValue":"false","parameterType":"Boolean","description":"Build header from the first row"}],"engineMeta":{"spark":"CSVSteps.csvStringToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"15889487-fd1c-4c44-b8eb-973c12f91fae","displayName":"Creates an HttpRestClient","description":"This step will build an HttpRestClient using a host url and optional authorization object","type":"Pipeline","category":"API","params":[{"type":"text","name":"hostUrl","required":true,"parameterType":"String","description":"The URL to connect including port"},{"type":"text","name":"authorization","required":false,"parameterType":"com.acxiom.pipeline.api.Authorization","description":"The optional authorization class to use when making connections"},{"type":"boolean","name":"allowSelfSignedCertificates","required":false,"parameterType":"Boolean","description":"Flag to allow using self signed certificates for http calls"}],"engineMeta":{"spark":"ApiSteps.createHttpRestClient","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.api.HttpRestClient"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"fcfd4b91-9a9c-438c-8afa-9f14c1e52a82","displayName":"Creates an HttpRestClient from protocol, host and port","description":"This step will build an HttpRestClient using url parts and optional authorization object","type":"Pipeline","category":"API","params":[{"type":"text","name":"protocol","required":true,"parameterType":"String","description":"The protocol to use when constructing the URL"},{"type":"text","name":"host","required":true,"parameterType":"String","description":"The host name to use when constructing the URL"},{"type":"text","name":"port","required":true,"parameterType":"Int","description":"The port to use when constructing the URL"},{"type":"text","name":"authorization","required":false,"parameterType":"com.acxiom.pipeline.api.Authorization","description":"The optional authorization class to use when making connections"}],"engineMeta":{"spark":"ApiSteps.createHttpRestClientFromParameters","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.api.HttpRestClient"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"b59f0486-78aa-4bd4-baf5-5c7d7c648ff0","displayName":"Check Path Exists","description":"Checks the path to determine whether it exists or not.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to verify"}],"engineMeta":{"spark":"ApiSteps.exists","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"7521ac47-84ec-4e50-b087-b9de4bf6d514","displayName":"Get the last modified date","description":"Gets the last modified date for the provided path","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource to get the last modified date"}],"engineMeta":{"spark":"ApiSteps.getLastModifiedDate","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.util.Date"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"fff7f7b6-5d9a-40b3-8add-6432552920a8","displayName":"Get Path Content Length","description":"Get the size of the content at the given path.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource to get the content length"}],"engineMeta":{"spark":"ApiSteps.getContentLength","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Long"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"dd351d47-125d-47fa-bafd-203bebad82eb","displayName":"Get Path Headers","description":"Get the headers for the content at the given path.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to get the headers"}],"engineMeta":{"spark":"ApiSteps.getHeaders","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Map[String,List[String]]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"532f72dd-8443-481d-8406-b74cdc08e342","displayName":"Delete Content","description":"Attempts to delete the provided path..","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to delete"}],"engineMeta":{"spark":"ApiSteps.delete","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3b91e6e8-ec18-4468-9089-8474f4b4ba48","displayName":"GET String Content","description":"Retrieves the value at the provided path as a string.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to resource"}],"engineMeta":{"spark":"ApiSteps.getStringContent","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"34c2fc9a-2502-4c79-a0cb-3f866a0a0d6e","displayName":"POST String Content","description":"POSTs the provided string to the provided path using the content type and returns the response as a string.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to post the content"},{"type":"text","name":"content","required":true,"parameterType":"String","description":"The content to post"},{"type":"text","name":"contentType","required":false,"parameterType":"String","description":"The content type being sent to the path"}],"engineMeta":{"spark":"ApiSteps.postStringContent","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"49ae38b3-cb41-4153-9111-aa6aacf6721d","displayName":"PUT String Content","description":"PUTs the provided string to the provided path using the content type and returns the response as a string.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to post the content"},{"type":"text","name":"content","required":true,"parameterType":"String","description":"The content to put"},{"type":"text","name":"contentType","required":false,"parameterType":"String","description":"The content type being sent to the path"}],"engineMeta":{"spark":"ApiSteps.putStringContent","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"99b20c23-722f-4862-9f47-bc9f72440ae6","displayName":"GET Input Stream","description":"Creates a buffered input stream for the provided path","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource"},{"type":"text","name":"bufferSize","required":false,"parameterType":"Int","description":"The size of buffer to use with the stream"}],"engineMeta":{"spark":"ApiSteps.getInputStream","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.io.InputStream"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f4120b1c-91df-452f-9589-b77f8555ba44","displayName":"GET Output Stream","description":"Creates a buffered output stream for the provided path.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource"},{"type":"text","name":"bufferSize","required":false,"parameterType":"Int","description":"The size of buffer to use with the stream"}],"engineMeta":{"spark":"ApiSteps.getOutputStream","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.io.OutputStream"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"cdb332e3-9ea4-4c96-8b29-c1d74287656c","displayName":"Load table as DataFrame using JDBCOptions","description":"This step will load a table from the provided JDBCOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"jdbcOptions","required":true,"parameterType":"org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions","description":"The options to use when loading the DataFrame"}],"engineMeta":{"spark":"JDBCSteps.readWithJDBCOptions","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"72dbbfc8-bd1d-4ce4-ab35-28fa8385ea54","displayName":"Load table as DataFrame using StepOptions","description":"This step will load a table from the provided JDBCDataFrameReaderOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"object","name":"jDBCStepsOptions","required":true,"className":"com.acxiom.pipeline.steps.JDBCDataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.JDBCDataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"JDBCSteps.readWithStepOptions","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"dcc57409-eb91-48c0-975b-ca109ba30195","displayName":"Load table as DataFrame","description":"This step will load a table from the provided jdbc information","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"url","required":true,"parameterType":"String","description":"A valid jdbc url"},{"type":"text","name":"table","required":true,"parameterType":"String","description":"A table name or subquery"},{"type":"text","name":"predicates","required":false,"parameterType":"List[String]","description":"Optional predicates used for partitioning"},{"type":"text","name":"connectionProperties","required":false,"parameterType":"Map[String,String]","description":"Optional properties for the jdbc connection"}],"engineMeta":{"spark":"JDBCSteps.readWithProperties","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"c9fddf52-34b1-4216-a049-10c33ccd24ab","displayName":"Write DataFrame to table using JDBCOptions","description":"This step will write a DataFrame as a table using JDBCOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to be written"},{"type":"text","name":"jdbcOptions","required":true,"parameterType":"org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions","description":"Options for configuring the JDBC connection"},{"type":"text","name":"saveMode","required":false,"parameterType":"String","description":"The value for the mode option. Defaulted to Overwrite"}],"engineMeta":{"spark":"JDBCSteps.writeWithJDBCOptions","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"77ffcd02-fbd0-4f79-9b35-ac9dc5fb7190","displayName":"Write DataFrame to table","description":"This step will write a DataFrame to a table using the provided properties","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to be written"},{"type":"text","name":"url","required":true,"parameterType":"String","description":"A valid jdbc url"},{"type":"text","name":"table","required":true,"parameterType":"String","description":"A table name or subquery"},{"type":"text","name":"connectionProperties","required":false,"parameterType":"Map[String,String]","description":"Optional properties for the jdbc connection"},{"type":"text","name":"saveMode","required":false,"parameterType":"String","description":"The value for the mode option. Defaulted to Overwrite"}],"engineMeta":{"spark":"JDBCSteps.writeWithProperties","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3d6b77a1-52c2-49ba-99a0-7ec773dac696","displayName":"Write DataFrame to JDBC table","description":"This step will write a DataFrame to a table using the provided JDBCDataFrameWriterOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to be written"},{"type":"object","name":"jDBCStepsOptions","required":true,"className":"com.acxiom.pipeline.steps.JDBCDataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.JDBCDataFrameWriterOptions","description":"Options for the JDBC connect and spark DataFrameWriter"}],"engineMeta":{"spark":"JDBCSteps.writeWithStepOptions","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"713fff3d-d407-4970-89ae-7844e6fc60e3","displayName":"Get JDBC Connection","description":"Get a jdbc connection.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"url","required":true,"parameterType":"String","description":"A valid jdbc url"},{"type":"text","name":"properties","required":false,"parameterType":"Map[String,String]","description":"Optional properties for the jdbc connection"}],"engineMeta":{"spark":"JDBCSteps.getConnection","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.sql.Connection"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"549828be-3d96-4561-bf94-7ad420f9d203","displayName":"Execute Sql","description":"Execute a sql command using jdbc.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"sql","required":true,"parameterType":"String","description":"Sql command to execute"},{"type":"text","name":"connection","required":true,"parameterType":"java.sql.Connection","description":"An open jdbc connection"},{"type":"text","name":"parameters","required":false,"parameterType":"List[Any]","description":"Optional list of bind variables"}],"engineMeta":{"spark":"JDBCSteps.executeSql","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"9c8957a3-899e-4f32-830e-d120b1917aa1","displayName":"Close JDBC Connection","description":"Close a JDBC Connection.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"connection","required":true,"parameterType":"java.sql.Connection","description":"An open jdbc connection"}],"engineMeta":{"spark":"JDBCSteps.closeConnection","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3464dc85-5111-40fc-9bfb-1fd6fc8a2c17","displayName":"Convert JSON String to Map","description":"This step will convert the provided JSON string into a Map that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonString","required":true,"parameterType":"String","description":"The JSON string to convert to a map"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonStringToMap","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Map[String,Any]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f4d19691-779b-4962-a52b-ee5d9a99068e","displayName":"Convert JSON Map to JSON String","description":"This step will convert the provided JSON map into a JSON string that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonMap","required":true,"parameterType":"Map[String,Any]","description":"The JSON map to convert to a JSON string"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonMapToString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"1f23eb37-98ee-43c2-ac78-17b04db3cc8d","displayName":"Convert object to JSON String","description":"This step will convert the provided object into a JSON string that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"obj","required":true,"parameterType":"AnyRef","description":"The object to convert to a JSON string"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.objectToJsonString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"880c5151-f7cd-40bb-99f2-06dbb20a6523","displayName":"Convert JSON String to object","description":"This step will convert the provided JSON string into an object that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonString","required":true,"parameterType":"String","description":"The JSON string to convert to an object"},{"type":"text","name":"objectName","required":true,"parameterType":"String","description":"The fully qualified class name of the object"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonStringToObject","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Any"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"68958a29-aab5-4f7e-9ffd-af99c33c512b","displayName":"Convert JSON String to Schema","description":"This step will convert the provided JSON string into a Schema that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"schema","required":true,"parameterType":"String","description":"The JSON string to convert to a Schema"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonStringToSchema","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.Schema"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"cf4e9e6c-98d6-4a14-ae74-52322782c504","displayName":"Convert JSON String to DataFrame","description":"This step will convert the provided JSON string into a DataFrame that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonString","required":true,"parameterType":"String","description":"The JSON string to convert to a DataFrame"}],"engineMeta":{"spark":"JSONSteps.jsonStringToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"d5cd835e-5e8f-49c0-9706-746d5a4d7b3a","displayName":"Convert JSON String Dataset to DataFrame","description":"This step will convert the provided JSON string Dataset into a DataFrame that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"dataset","required":true,"parameterType":"org.apache.spark.sql.Dataset[String]","description":"The dataset containing JSON strings"},{"type":"object","name":"dataFrameReaderOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The JSON parsing options"}],"engineMeta":{"spark":"JSONSteps.jsonDatasetToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f3891201-5138-4cab-aebc-bcc319228543","displayName":"Build JSON4S Formats","description":"This step will build a json4s Formats object that can be used to override the default","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"customSerializers","required":false,"parameterType":"List[com.acxiom.pipeline.applications.ClassInfo]","description":"List of custom serializer classes"},{"type":"text","name":"enumIdSerializers","required":false,"parameterType":"List[com.acxiom.pipeline.applications.ClassInfo]","description":"List of Enumeration classes to serialize by id"},{"type":"text","name":"enumNameSerializers","required":false,"parameterType":"List[com.acxiom.pipeline.applications.ClassInfo]","description":"List of Enumeration classes to serialize by name"}],"engineMeta":{"spark":"JSONSteps.buildJsonFormats","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.json4s.Formats"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"b5485d97-d4e8-41a6-8af7-9ce79a435140","displayName":"To String","description":"Returns the result of the toString method, can unwrap options","type":"Pipeline","category":"String","params":[{"type":"text","name":"value","required":true,"parameterType":"Any","description":"The value to convert"},{"type":"boolean","name":"unwrapOption","required":false,"parameterType":"Boolean","description":"Boolean indicating whether to unwrap the value from an Option prior to calling toString"}],"engineMeta":{"spark":"StringSteps.toString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"78e817ec-2bf2-4cbe-acba-e5bc9bdcffc5","displayName":"List To String","description":"Returns the result of the mkString method","type":"Pipeline","category":"String","params":[{"type":"text","name":"list","required":true,"parameterType":"List[Any]","description":"The list to convert"},{"type":"text","name":"separator","required":false,"parameterType":"String","description":"Separator character to use when making the string"},{"type":"boolean","name":"unwrapOptions","required":false,"parameterType":"Boolean","description":"Boolean indicating whether to unwrap each value from an Option"}],"engineMeta":{"spark":"StringSteps.listToString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"fcd6b5fe-08ed-4cfd-acfe-eb676d7f4ecd","displayName":"To Lowercase","description":"Returns a lowercase string","type":"Pipeline","category":"String","params":[{"type":"text","name":"value","required":true,"parameterType":"String","description":"The value to lowercase"}],"engineMeta":{"spark":"StringSteps.toLowerCase","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"2f31ebf1-4ae2-4e04-9b29-4802cac8a198","displayName":"To Uppercase","description":"Returns an uppercase string","type":"Pipeline","category":"String","params":[{"type":"text","name":"value","required":true,"parameterType":"String","description":"The value to uppercase"}],"engineMeta":{"spark":"StringSteps.toUpperCase","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"96b7b521-5304-4e63-8435-63d84a358368","displayName":"String Split","description":"Returns a list of strings split off of the given string","type":"Pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to split"},{"type":"text","name":"regex","required":true,"parameterType":"String","description":"Regex to use when splitting the string"},{"type":"integer","name":"limit","required":false,"parameterType":"Int","description":"Max number elements to return in the list"}],"engineMeta":{"spark":"StringSteps.stringSplit","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"List[String]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f75abedd-4aee-4979-8d56-ea7b0c1a86e1","displayName":"Substring","description":"Returns a substring","type":"Pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to parse"},{"type":"text","name":"begin","required":true,"parameterType":"Int","description":"The beginning index"},{"type":"integer","name":"end","required":false,"parameterType":"Int","description":"The end index"}],"engineMeta":{"spark":"StringSteps.substring","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3fabf9ec-5383-4eb3-81af-6092ab7c370d","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","category":"Decision","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"parameterType":"String","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"parameterType":"Boolean","description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"ff0562f5-2917-406d-aa78-c5d49ba6b99f","displayName":"String Matches","description":"Return whether string matches a given regex","type":"branch","category":"Decision","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to match"},{"type":"text","name":"regex","required":true,"parameterType":"String","description":"Regex to use for the match"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"StringSteps.stringMatches","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"416baf4e-a1dd-49fc-83a9-0f41b77e57b7","displayName":"String Replace All","description":"Perform a literal or regex replacement on a string","type":"pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to modify"},{"type":"text","name":"matchString","required":true,"parameterType":"String","description":"The string to match"},{"type":"text","name":"replacement","required":false,"parameterType":"String","description":"The replacement string"},{"type":"boolean","name":"literal","required":false,"parameterType":"Boolean","description":"Perform \\'literal\\' match replacement"}],"engineMeta":{"spark":"StringSteps.stringReplaceAll","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"95438b82-8d50-41da-8094-c92449b9e7df","displayName":"String Replace First","description":"Perform a literal or regex replacement on the first occurrence in a string","type":"pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to modify"},{"type":"text","name":"matchString","required":true,"parameterType":"String","description":"The string to match"},{"type":"text","name":"replacement","required":false,"parameterType":"String","description":"The replacement string"},{"type":"boolean","name":"literal","required":false,"parameterType":"Boolean","description":"Perform \\'literal\\' match replacement"}],"engineMeta":{"spark":"StringSteps.stringReplaceFirst","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"86c84fa3-ad45-4a49-ac05-92385b8e9572","displayName":"Get Credential","description":"This step provides access to credentials through the CredentialProvider","type":"Pipeline","category":"Credentials","params":[{"type":"text","name":"credentialName","required":true,"parameterType":"String","description":"The dataset containing CSV strings"}],"engineMeta":{"spark":"CredentialSteps.getCredential","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.Credential"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"219c787a-f502-4efc-b15d-5beeff661fc0","displayName":"Map a DataFrame to an existing DataFrame","description":"This step maps a new DataFrame to an existing DataFrame to make them compatible","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"inputDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame that needs to be modified"},{"type":"text","name":"destinationDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame that the new data needs to map to"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations","description":"The object with transform, alias, and filter logic details"},{"type":"boolean","name":"addNewColumns","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.mapToDestinationDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"8f9c08ea-4882-4265-bac7-2da3e942758f","displayName":"Map a DataFrame to a pre-defined Schema","description":"This step maps a new DataFrame to a pre-defined spark schema","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"inputDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame that needs to be modified"},{"type":"object","name":"destinationSchema","required":true,"className":"com.acxiom.pipeline.steps.Schema","parameterType":"com.acxiom.pipeline.steps.Schema","description":"The schema that the new data should map to"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations","description":"The object with transform, alias, and filter logic details"},{"type":"boolean","name":"addNewColumns","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.mapDataFrameToSchema","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3ee74590-9131-43e1-8ee8-ad320482a592","displayName":"Merge a DataFrame to an existing DataFrame","description":"This step merges two DataFrames to create a single DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"inputDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The first DataFrame"},{"type":"text","name":"destinationDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The second DataFrame used as the driver"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations","description":"The object with transform, alias, and filter logic details"},{"type":"boolean","name":"addNewColumns","required":false,"parameterType":"Boolean"},{"type":"boolean","name":"distinct","required":false,"defaultValue":"true","parameterType":"Boolean","description":"Flag to determine whether a distinct union should be performed"}],"engineMeta":{"spark":"TransformationSteps.mergeDataFrames","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"ac3dafe4-e6ee-45c9-8fc6-fa7f918cf4f2","displayName":"Modify or Create Columns using Transforms Provided","description":"This step transforms existing columns and/or adds new columns to an existing dataframe using expressions provided","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The input DataFrame"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations"}],"engineMeta":{"spark":"TransformationSteps.applyTransforms","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3e2da5a8-387d-49b1-be22-c03764fb0fde","displayName":"Select Expressions","description":"Select each provided expresion from a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to select from"},{"type":"text","name":"expressions","required":true,"parameterType":"List[String]","description":"List of expressions to select"}],"engineMeta":{"spark":"TransformationSteps.selectExpressions","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"1e0a234a-8ae5-4627-be6d-3052b33d9014","displayName":"Add Column","description":"Add a new column to a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to add to"},{"type":"text","name":"columnName","required":true,"parameterType":"String","description":"The name of the new column"},{"type":"text","name":"expression","required":true,"parameterType":"String","description":"The expression used for the column"},{"type":"boolean","name":"standardizeColumnName","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.addColumn","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"08c9c5a9-a10d-477e-a702-19bd24889d1e","displayName":"Add Columns","description":"Add multiple new columns to a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to add to"},{"type":"text","name":"columns","required":true,"parameterType":"Map[String,String]","description":"A map of column names and expressions"},{"type":"boolean","name":"standardizeColumnNames","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.addColumns","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"42c328ac-a6bd-49ca-b597-b706956d294c","displayName":"Flatten a DataFrame","description":"This step will flatten all nested fields contained in a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to flatten"},{"type":"text","name":"separator","required":false,"defaultValue":"_","parameterType":"String","description":"Separator to place between nested field names"},{"type":"text","name":"fieldList","required":false,"parameterType":"List[String]","description":"List of fields to flatten. Will flatten all fields if left empty"},{"type":"integer","name":"depth","required":false,"parameterType":"Int","description":"How deep should we traverse when flattening."}],"engineMeta":{"spark":"TransformationSteps.flattenDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[_]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"a981080d-714c-4d36-8b09-d95842ec5655","displayName":"Standardize Column Names on a DataFrame","description":"This step will standardize columns names on existing DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.DataFrame"}],"engineMeta":{"spark":"TransformationSteps.standardizeColumnNames","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"541c4f7d-3524-4d53-bbd9-9f2cfd9d1bd1","displayName":"Save a Dataframe to a TempView","description":"This step stores an existing dataframe to a TempView to be used in future queries in the session","type":"Pipeline","category":"Query","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The dataframe to store"},{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to create (optional, random name will be created if not provided)"}],"engineMeta":{"spark":"QuerySteps.dataFrameToTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"71b71ef3-eaa7-4a1f-b3f3-603a1a54846d","displayName":"Create a TempView from a Query","description":"This step runs a SQL statement against existing TempViews from this session and returns a new TempView","type":"Pipeline","category":"Query","params":[{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"},{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to create (optional, random name will be created if not provided)"}],"engineMeta":{"spark":"QuerySteps.queryToTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"61378ed6-8a4f-4e6d-9c92-6863c9503a54","displayName":"Create a DataFrame from a Query","description":"This step runs a SQL statement against existing TempViews from this session and returns a new DataFrame","type":"Pipeline","category":"Query","params":[{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"}],"engineMeta":{"spark":"QuerySteps.queryToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"57b0e491-e09b-4428-aab2-cebe1f217eda","displayName":"Create a DataFrame from an Existing TempView","description":"This step pulls an existing TempView from this session into a new DataFrame","type":"Pipeline","category":"Query","params":[{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to use"}],"engineMeta":{"spark":"QuerySteps.tempViewToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"648f27aa-6e3b-44ed-a093-bc284783731b","displayName":"Create a TempView from a DataFrame Query","description":"This step runs a SQL statement against an existing DataFrame from this session and returns a new TempView","type":"Pipeline","category":"Query","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The dataframe to query"},{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"},{"type":"text","name":"inputViewName","required":true,"parameterType":"String","description":"The name to use when creating the view representing the input dataframe (same name used in query)"},{"type":"text","name":"outputViewName","required":false,"parameterType":"String","description":"The name of the view to create (optional, random name will be created if not provided)"}],"engineMeta":{"spark":"QuerySteps.dataFrameQueryToTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"dfb8a387-6245-4b1c-ae6c-94067eb83962","displayName":"Create a DataFrame from a DataFrame Query","description":"This step runs a SQL statement against an existing DataFrame from this session and returns a new DataFrame","type":"Pipeline","category":"Query","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The dataframe to query"},{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"},{"type":"text","name":"inputViewName","required":true,"parameterType":"String","description":"The name to use when creating the view representing the input dataframe (same name used in query)"}],"engineMeta":{"spark":"QuerySteps.dataFrameQueryToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"c88de095-14e0-4c67-8537-0325127e2bd2","displayName":"Cache an exising TempView","description":"This step will cache an existing TempView","type":"Pipeline","category":"Query","params":[{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to cache"}],"engineMeta":{"spark":"QuerySteps.cacheTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"0342654c-2722-56fe-ba22-e342169545af","displayName":"Copy (auto buffering)","description":"Copy the contents of the source path to the destination path. This function will call connect on both FileManagers.","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"c40169a3-1e77-51ab-9e0a-3f24fb98beef","displayName":"Copy (basic buffering)","description":"Copy the contents of the source path to the destination path using buffer sizes. This function will call connect on both FileManagers.","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to"},{"type":"text","name":"inputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for reading data during copy"},{"type":"text","name":"outputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for writing data during copy"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f5a24db0-e91b-5c88-8e67-ab5cff09c883","displayName":"Copy (advanced buffering)","description":"Copy the contents of the source path to the destination path using full buffer sizes. This function will call connect on both FileManagers.","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to"},{"type":"text","name":"inputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for reading data during copy"},{"type":"text","name":"outputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for writing data during copy"},{"type":"text","name":"copyBufferSize","required":true,"parameterType":"Int","description":"The intermediate buffer size to use during copy"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"1af68ab5-a3fe-4afb-b5fa-34e52f7c77f5","displayName":"Compare File Sizes","description":"Compare the file sizes of the source and destination paths","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to the source"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to th destination"}],"engineMeta":{"spark":"FileManagerSteps.compareFileSizes","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Int"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"bf2c4df8-a215-480b-87d8-586984e04189","displayName":"Delete (file)","description":"Delete a file","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The FileManager"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the file being deleted"}],"engineMeta":{"spark":"FileManagerSteps.deleteFile","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3d1e8519-690c-55f0-bd05-1e7b97fb6633","displayName":"Disconnect a FileManager","description":"Disconnects a FileManager from the underlying file system","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The file manager to disconnect"}],"engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"259a880a-3e12-4843-9f02-2cfc2a05f576","displayName":"Create a FileManager","description":"Creates a FileManager using the provided FileConnector","type":"Pipeline","category":"Connectors","params":[{"type":"text","name":"fileConnector","required":true,"parameterType":"com.acxiom.pipeline.connectors.FileConnector","description":"The FileConnector to use to create the FileManager implementation"}],"engineMeta":{"spark":"FileManagerSteps.getFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.FileManager"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"9d467cb0-8b3d-40a0-9ccd-9cf8c5b6cb38","displayName":"Create SFTP FileManager","description":"Simple function to generate the SFTPFileManager for the remote SFTP file system","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"hostName","required":true,"parameterType":"String","description":"The name of the host to connect"},{"type":"text","name":"username","required":false,"parameterType":"String","description":"The username used for connection"},{"type":"text","name":"password","required":false,"parameterType":"String","description":"The password used for connection"},{"type":"integer","name":"port","required":false,"parameterType":"Int","description":"The optional port if other than 22"},{"type":"boolean","name":"strictHostChecking","required":false,"parameterType":"Boolean","description":"Option to automatically add keys to the known_hosts file. Default is false."}],"engineMeta":{"spark":"SFTPSteps.createFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.SFTPFileManager"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"22fcc0e7-0190-461c-a999-9116b77d5919","displayName":"Build a DataFrameReader Object","description":"This step will build a DataFrameReader object that can be used to read a file into a dataframe","type":"Pipeline","category":"InputOutput","params":[{"type":"object","name":"dataFrameReaderOptions","required":true,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"DataFrameSteps.getDataFrameReader","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrameReader"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"66a451c8-ffbd-4481-9c37-71777c3a240f","displayName":"Load Using DataFrameReader","description":"This step will load a DataFrame given a dataFrameReader.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrameReader","required":true,"parameterType":"org.apache.spark.sql.DataFrameReader","description":"The DataFrameReader to use when creating the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.load","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"d7cf27e6-9ca5-4a73-a1b3-d007499f235f","displayName":"Load DataFrame","description":"This step will load a DataFrame given a DataFrameReaderOptions object.","type":"Pipeline","category":"InputOutput","params":[{"type":"object","name":"dataFrameReaderOptions","required":true,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The DataFrameReaderOptions to use when creating the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.loadDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"8a00dcf8-e6a9-4833-871e-c1f3397ab378","displayName":"Build a DataFrameWriter Object","description":"This step will build a DataFrameWriter object that can be used to write a file into a dataframe","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to use when creating the DataFrameWriter"},{"type":"object","name":"options","required":true,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The DataFrameWriterOptions to use when writing the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.getDataFrameWriter","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrameWriter[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"9aa6ae9f-cbeb-4b36-ba6a-02eee0a46558","displayName":"Save Using DataFrameWriter","description":"This step will save a DataFrame given a dataFrameWriter[Row].","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrameWriter","required":true,"parameterType":"org.apache.spark.sql.DataFrameWriter[_]","description":"The DataFrameWriter to use when saving"}],"engineMeta":{"spark":"DataFrameSteps.save","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"e5ac3671-ee10-4d4e-8206-fec7effdf7b9","displayName":"Save DataFrame","description":"This step will save a DataFrame given a DataFrameWriterOptions object.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to save"},{"type":"object","name":"dataFrameWriterOptions","required":true,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The DataFrameWriterOptions to use for saving"}],"engineMeta":{"spark":"DataFrameSteps.saveDataFrame","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"fa05a970-476d-4617-be4d-950cfa65f2f8","displayName":"Persist DataFrame","description":"Persist a DataFrame to provided storage level.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to persist"},{"type":"text","name":"storageLevel","required":false,"parameterType":"String","description":"The optional storage mechanism to use when persisting the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.persistDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"e6fe074e-a1fa-476f-9569-d37295062186","displayName":"Unpersist DataFrame","description":"Unpersist a DataFrame.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to unpersist"},{"type":"boolean","name":"blocking","required":false,"parameterType":"Boolean","description":"Optional flag to indicate whether to block while unpersisting"}],"engineMeta":{"spark":"DataFrameSteps.unpersistDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"71323226-bcfd-4fa1-bf9e-24e455e41144","displayName":"RepartitionDataFrame","description":"Repartition a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to repartition"},{"type":"text","name":"partitions","required":true,"parameterType":"Int","description":"The number of partitions to use"},{"type":"boolean","name":"rangePartition","required":false,"parameterType":"Boolean","description":"Flag indicating whether to repartition by range. This takes precedent over the shuffle flag"},{"type":"boolean","name":"shuffle","required":false,"parameterType":"Boolean","description":"Flag indicating whether to perform a normal partition"},{"type":"text","name":"partitionExpressions","required":false,"parameterType":"List[String]","description":"The partition expressions to use"}],"engineMeta":{"spark":"DataFrameSteps.repartitionDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"5e0358a0-d567-5508-af61-c35a69286e4e","displayName":"Javascript Step","description":"Executes a script and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"javascript","parameterType":"String","description":"Javascript to execute"}],"engineMeta":{"spark":"JavascriptSteps.processScript","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"570c9a80-8bd1-5f0c-9ae0-605921fe51e2","displayName":"Javascript Step with single object provided","description":"Executes a script with single object provided and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"javascript","parameterType":"String","description":"Javascript script to execute"},{"type":"text","name":"value","required":true,"parameterType":"Any","description":"Value to bind to the script"}],"engineMeta":{"spark":"JavascriptSteps.processScriptWithValue","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f92d4816-3c62-4c29-b420-f00994bfcd86","displayName":"Javascript Step with map of objects provided","description":"Executes a script with map of objects provided and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"javascript","parameterType":"String"},{"type":"text","name":"values","required":true,"parameterType":"Map[String,Any]","description":"Map of name/value pairs to bind to the script"},{"type":"boolean","name":"unwrapOptions","required":false,"parameterType":"Boolean","description":"Flag to control option unwrapping behavior"}],"engineMeta":{"spark":"JavascriptSteps.processScriptWithValues","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]}],"pkgObjs":[{"id":"com.acxiom.pipeline.steps.JDBCDataFrameReaderOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"JDBC Data Frame Reader Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"url\":{\"type\":\"string\"},\"table\":{\"type\":\"string\"},\"predicates\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"readerOptions\":{\"$ref\":\"#/definitions/DataFrameReaderOptions\"}},\"definitions\":{\"DataFrameReaderOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}},\"Schema\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}}},\"Attribute\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"name\":{\"type\":\"string\"},\"dataType\":{\"$ref\":\"#/definitions/AttributeType\"}}},\"AttributeType\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"baseType\":{\"type\":\"string\"},\"valueType\":{\"$ref\":\"#/definitions/AttributeType\"},\"nameType\":{\"$ref\":\"#/definitions/AttributeType\"},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}}}}"},{"id":"com.acxiom.pipeline.steps.DataFrameWriterOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Data Frame Writer Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"saveMode\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"bucketingOptions\":{\"$ref\":\"#/definitions/BucketingOptions\"},\"partitionBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"sortBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}},\"definitions\":{\"BucketingOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"numBuckets\":{\"type\":\"integer\"},\"columns\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}},\"required\":[\"numBuckets\"]}}}","template":"{\"form\":[{\"type\":\"select\",\"key\":\"format\",\"templateOptions\":{\"label\":\"Format\",\"placeholder\":\"\",\"valueProp\":\"value\",\"options\":[{\"value\":\"csv\",\"name\":\"CSV\"},{\"value\":\"json\",\"name\":\"JSON\"},{\"value\":\"parquet\",\"name\":\"Parquet\"},{\"value\":\"orc\",\"name\":\"Orc\"},{\"value\":\"text\",\"name\":\"Text\"}],\"labelProp\":\"name\",\"focus\":false,\"_flatOptions\":true,\"disabled\":false}},{\"key\":\"options.encoding\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Encoding\",\"placeholder\":\"\",\"focus\":false},\"expressionProperties\":{\"templateOptions.disabled\":\"!model.format\"}},{\"key\":\"saveMode\",\"type\":\"select\",\"defaultValue\":false,\"templateOptions\":{\"label\":\"Save Mode\",\"placeholder\":\"\",\"valueProp\":\"value\",\"options\":[{\"value\":\"OVERWRITE\",\"name\":\"Overwrite\"},{\"value\":\"append\",\"name\":\"Append\"},{\"value\":\"ignore\",\"name\":\"Ignore\"},{\"value\":\"error\",\"name\":\"Error\"},{\"value\":\"errorifexists\",\"name\":\"Error If Exists\"}],\"labelProp\":\"name\",\"focus\":false,\"_flatOptions\":true,\"disabled\":false}},{\"key\":\"partitionBy\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Partition By Columns\",\"placeholder\":\"Add column names to use during partitioning\"}},{\"key\":\"sortBy\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Sort By Columns\",\"placeholder\":\"Add column names to use during sorting\"}},{\"key\":\"bucketingOptions\",\"wrappers\":[\"panel\"],\"templateOptions\":{\"label\":\"Bucketing Options\"},\"fieldGroup\":[{\"key\":\"numBuckets\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Number of Buckets\",\"type\":\"number\"}},{\"key\":\"columns\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Bucket Columns\",\"placeholder\":\"Add column names to use during bucketing\"}}]},{\"key\":\"options.sep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.lineSep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Line Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'text' || model.format === 'json' ? false : true\"},{\"key\":\"options.escapeQuotes\",\"hideExpression\":\"model.format === 'csv' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Escape Quotes?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.quote\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Quote\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.escape\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Escape Character\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"}]}"},{"id":"com.acxiom.pipeline.steps.JDBCDataFrameWriterOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"JDBC Data Frame Writer Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"url\":{\"type\":\"string\"},\"table\":{\"type\":\"string\"},\"writerOptions\":{\"$ref\":\"#/definitions/DataFrameWriterOptions\"}},\"definitions\":{\"DataFrameWriterOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"saveMode\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"bucketingOptions\":{\"$ref\":\"#/definitions/BucketingOptions\"},\"partitionBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"sortBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}}},\"BucketingOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"numBuckets\":{\"type\":\"integer\"},\"columns\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}},\"required\":[\"numBuckets\"]}}}"},{"id":"com.acxiom.pipeline.steps.Transformations","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Transformations\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"columnDetails\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/ColumnDetails\"}},\"filter\":{\"type\":\"string\"},\"standardizeColumnNames\":{}},\"definitions\":{\"ColumnDetails\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"outputField\":{\"type\":\"string\"},\"inputAliases\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"expression\":{\"type\":\"string\"}}}}}","template":"{\"form\":[{\"key\":\"filter\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Filter\"}},{\"key\":\"standardizeColumnNames\",\"type\":\"checkbox\",\"defaultValue\":false,\"templateOptions\":{\"floatLabel\":\"always\",\"align\":\"start\",\"label\":\"Standardize Column Names?\",\"hideFieldUnderline\":true,\"color\":\"accent\",\"placeholder\":\"\",\"focus\":false,\"hideLabel\":true,\"disabled\":false,\"indeterminate\":true}},{\"fieldArray\":{\"fieldGroup\":[{\"key\":\"outputField\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Output Field\"}},{\"key\":\"inputAliases\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Input Alias\"}},{\"key\":\"expression\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Expression (Optional)\"}}]},\"key\":\"columnDetails\",\"wrappers\":[\"panel\"],\"type\":\"repeat\",\"templateOptions\":{\"label\":\"Column Details\"}}]}"},{"id":"com.acxiom.pipeline.steps.DataFrameReaderOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Data Frame Reader Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"schema\":{\"$ref\":\"#/definitions/Schema\"}},\"definitions\":{\"Schema\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}}},\"Attribute\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"name\":{\"type\":\"string\"},\"dataType\":{\"$ref\":\"#/definitions/AttributeType\"}}},\"AttributeType\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"baseType\":{\"type\":\"string\"},\"valueType\":{\"$ref\":\"#/definitions/AttributeType\"},\"nameType\":{\"$ref\":\"#/definitions/AttributeType\"},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}}}}","template":"{\"form\":[{\"type\":\"select\",\"key\":\"format\",\"templateOptions\":{\"label\":\"Format\",\"placeholder\":\"\",\"valueProp\":\"value\",\"options\":[{\"value\":\"csv\",\"name\":\"CSV\"},{\"value\":\"json\",\"name\":\"JSON\"},{\"value\":\"parquet\",\"name\":\"Parquet\"},{\"value\":\"orc\",\"name\":\"Orc\"},{\"value\":\"text\",\"name\":\"Text\"}],\"labelProp\":\"name\",\"focus\":false,\"_flatOptions\":true,\"disabled\":false}},{\"key\":\"options.encoding\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Encoding\",\"placeholder\":\"\",\"focus\":false},\"expressionProperties\":{\"templateOptions.disabled\":\"!model.format\"}},{\"key\":\"options.multiLine\",\"hideExpression\":\"model.format === 'csv' || model.format === 'json' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Multiline?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.header\",\"hideExpression\":\"model.format === 'csv' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Skip Header?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.sep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.lineSep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Line Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'text' || model.format === 'json' ? false : true\"},{\"key\":\"options.quote\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Quote\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.escape\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Escape Character\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.primitivesAsString\",\"hideExpression\":\"model.format === 'json' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Primitive As String?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.inferSchema\",\"hideExpression\":\"model.format === 'csv' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Infer Schema?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"expressionProperties\":{\"templateOptions.disabled\":\"model.options.inferSchema || model.format === 'json' ? false : true\"},\"key\":\"options.samplingRatio\",\"hideExpression\":\"model.format === 'csv' || model.format === 'json' ? false : true\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Sampling Ration\",\"type\":\"number\"}}]}"},{"id":"com.acxiom.pipeline.steps.Schema","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Schema\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}},\"definitions\":{\"Attribute\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"name\":{\"type\":\"string\"},\"dataType\":{\"$ref\":\"#/definitions/AttributeType\"}}},\"AttributeType\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"baseType\":{\"type\":\"string\"},\"valueType\":{\"$ref\":\"#/definitions/AttributeType\"},\"nameType\":{\"$ref\":\"#/definitions/AttributeType\"},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}},\"Schema\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}}}}}"}]} diff --git a/metalus-common/docs/dataoptions.md b/metalus-common/docs/dataoptions.md index 31f34103..1cfa29b5 100644 --- a/metalus-common/docs/dataoptions.md +++ b/metalus-common/docs/dataoptions.md @@ -10,6 +10,32 @@ This object represents options that will be translated into Spark settings at re * *format* - The file format to use. Defaulted to _parquet_. * *schema* - An optional _com.acxiom.pipeline.steps.Schema_ which will be applied to the DataFrame. * *options* - Optional map of settings to pass to the underlying Spark data source. +Example: +```json +{ + "format": "csv", + "options": { + "header": "true", + "delimiter": "," + }, + "schema": { + "attributes": [ + { + "name": "CUSTOMER_ID", + "dataType": { + "baseType": "Integer" + } + }, + { + "name": "FIRST_NAME", + "dataType": { + "baseType": "String" + } + } + ] + } +} +``` ## DataFrame Writer Options This object represents options that will be translated into Spark settings at write time. @@ -19,6 +45,33 @@ This object represents options that will be translated into Spark settings at wr * *bucketingOptions* - Optional BucketingOptions object for configuring Bucketing * *partitionBy* - Optional list of columns for partitioning. * *sortBy* - Optional list of columns for sorting. +Example: +```json +{ + "format": "parquet", + "saveMode": "Overwrite", + "bucketingOptions": {}, + "options": { + "escapeQuotes": false + }, + "schema": { + "attributes": [ + { + "name": "CUSTOMER_ID", + "dataType": { + "baseType": "Integer" + } + }, + { + "name": "FIRST_NAME", + "dataType": { + "baseType": "String" + } + } + ] + } +} +``` ## Bucketing Options This object represents the options used to bucket data when it is being written by Spark. diff --git a/metalus-gcp/docs/gcssteps.md b/metalus-gcp/docs/gcssteps.md index 109b7a98..a8616611 100644 --- a/metalus-gcp/docs/gcssteps.md +++ b/metalus-gcp/docs/gcssteps.md @@ -9,21 +9,21 @@ This function will write a given DataFrame to the provided path. Full parameter * **dataFrame** - A dataFrame to be written to GCS. * **path** - A GCS path where the data will be written. The bucket should be part of the path. * **credentials** - The credential map to use or none to use the system provided credentials. -* **options** - Optional DataFrameWriterOptions object to configure the DataFrameWriter +* **options** - Optional [DataFrameWriterOptions](../../metalus-common/docs/dataoptions.md#dataframe-writer-options) object to configure the DataFrameWriter ## Read From Path This function will read a file from the provided path into a DataFrame. Full parameter descriptions are listed below: * **path** - A GCS file path to read. The bucket should be part of the path. * **credentials** - The credential map to use or none to use the system provided credentials. -* **options** - Optional DataFrameReaderOptions object to configure the DataFrameReader +* **options** - Optional [DataFrameReaderOptions](../../metalus-common/docs/dataoptions.md#dataframe-reader-options) object to configure the DataFrameReader ## Read From Paths This function will read from each of the provided paths into a DataFrame. Full parameter descriptions are listed below: * **paths** - A list of GCS file paths to read. The bucket should be part of each path. * **credentials** - The credential map to use or none to use the system provided credentials. -* **options** - Optional DataFrameReaderOptions object to configure the DataFrameReader +* **options** - Optional [DataFrameReaderOptions](../../metalus-common/docs/dataoptions.md#dataframe-reader-options) object to configure the DataFrameReader ## Create FileManager This function will create a FileManager implementation that is useful for interacting with the the GCS file system. From ab0b4dadccd64ab224fb6cf4845e1140b2095e67 Mon Sep 17 00:00:00 2001 From: djfreels Date: Tue, 12 Oct 2021 15:25:45 -0400 Subject: [PATCH 21/24] Adding SparkConfigurationSteps --- .../docs/sparkconfigurationstepds.md | 39 ++++++ metalus-common/readme.md | 1 + .../steps/SparkConfigurationSteps.scala | 105 ++++++++++++++++ .../acxiom/pipeline/steps/CSVStepsTests.scala | 2 +- .../steps/SparkConfigurationStepsTests.scala | 118 ++++++++++++++++++ 5 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 metalus-common/docs/sparkconfigurationstepds.md create mode 100644 metalus-common/src/main/scala/com/acxiom/pipeline/steps/SparkConfigurationSteps.scala create mode 100644 metalus-common/src/test/scala/com/acxiom/pipeline/steps/SparkConfigurationStepsTests.scala diff --git a/metalus-common/docs/sparkconfigurationstepds.md b/metalus-common/docs/sparkconfigurationstepds.md new file mode 100644 index 00000000..800e0687 --- /dev/null +++ b/metalus-common/docs/sparkconfigurationstepds.md @@ -0,0 +1,39 @@ +[Documentation Home](../../docs/readme.md) | [Common Home](../readme.md) + +# SparkConfigurationSteps +This object exposes some basic functions to set configurations on the spark context at run time. + +##Set Spark Local Property +Set a local property on the current thread. + +* **key** - The key of the property to set. +* **value** - The value to set. Use None to remove the property. + +##Set Spark Local Properties +Set a local property on the current thread for each entry in the properties map. + +* **properties** - A Map where each entry will be set as a key/value pair. +* **replaceUnderScores** - Replace underscores in the property keys with periods. Enabled by default. + +##Set Hadoop Configuration Property +Set a property on the hadoop configuration. + +* **key** - The key of the property to set. +* **value** - The value to set. Use None to remove the property. + +##Set Hadoop Configuration Properties +Set a property on the hadoop configuration for each entry in the properties map. + +* **properties** - A Map where each entry will be set as a key/value pair. +* **replaceUnderScores** - Replace underscores in the property keys with periods. Enabled by default. + +##Set Job Group +Set a job group id and description to group all upcoming jobs on the current thread. + +* **groupId** - The name of the group. +* **description** - Description of the group. +* **interruptOnCancel** - When true, then job cancellation will result in Thread.interrupt() + getting called on the job's executor threads + +##Clear Job Group +Clears the current job group. diff --git a/metalus-common/readme.md b/metalus-common/readme.md index 838e1fea..27cb1ec6 100644 --- a/metalus-common/readme.md +++ b/metalus-common/readme.md @@ -20,6 +20,7 @@ using Spark. * [QuerySteps](docs/querysteps.md) * [ScalaSteps](docs/scalascriptsteps.md) * [SFTPSteps](docs/sftpsteps.md) +* [SparkConfigurationSteps](docs/sparkconfigurationstepds.md) * [StringSteps](docs/stringsteps.md) * [TransformationSteps](docs/transformationsteps.md) diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/SparkConfigurationSteps.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/SparkConfigurationSteps.scala new file mode 100644 index 00000000..96dce556 --- /dev/null +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/SparkConfigurationSteps.scala @@ -0,0 +1,105 @@ +package com.acxiom.pipeline.steps + +import com.acxiom.pipeline.PipelineContext +import com.acxiom.pipeline.annotations.{StepFunction, StepObject, StepParameter, StepParameters} + +import scala.annotation.tailrec + +@StepObject +object SparkConfigurationSteps { + + @StepFunction("5c4d2d01-da85-4e2e-a551-f5a65f83653a", + "Set Spark Local Property", + "Set a property on the spark context.", + "Pipeline", "Spark") + @StepParameters(Map("key" -> StepParameter(None, Some(true), None, None, None, None, Some("The name of the property to set")), + "value" -> StepParameter(None, Some(true), None, None, None, None, Some("The value to set")))) + def setLocalProperty(key: String, value: Any, pipelineContext: PipelineContext): Unit = { + setLocalProperties(Map(key -> value), Some(false), pipelineContext) + } + + @StepFunction("0b86b314-2657-4392-927c-e555af56b415", + "Set Spark Local Properties", + "Set each property on the spark context.", + "Pipeline", "Spark") + @StepParameters(Map("properties" -> StepParameter(None, Some(true), None, None, None, None, + Some("Map representing local properties to set")), + "replaceUnderScores" -> StepParameter(None, Some(false), Some("true"), None, None, None, + Some("When true, will replace underscores in the keys with periods")))) + def setLocalProperties(properties: Map[String, Any], replaceUnderScores: Option[Boolean] = None, pipelineContext: PipelineContext): Unit = { + val sc = pipelineContext.sparkSession.get.sparkContext + cleanseMap(properties, replaceUnderScores).foreach { + case (key, Some(value)) => sc.setLocalProperty(key, value.toString) + case (key, None) => sc.setLocalProperty(key, None.orNull) + case (key, value) => sc.setLocalProperty(key, value.toString) + } + } + + @StepFunction("c8c82365-e078-4a2a-99b8-0c0e20d8102d", + "Set Hadoop Configuration Properties", + "Set each property on the hadoop configuration.", + "Pipeline", "Spark") + @StepParameters(Map("properties" -> StepParameter(None, Some(true), None, None, None, None, + Some("Map representing local properties to set")), + "replaceUnderScores" -> StepParameter(None, Some(false), Some("true"), None, None, None, + Some("When true, will replace underscores in the keys with periods")))) + def setHadoopConfigurationProperties(properties: Map[String, Any], replaceUnderScores: Option[Boolean] = None, + pipelineContext: PipelineContext): Unit = { + val hc = pipelineContext.sparkSession.get.sparkContext.hadoopConfiguration + cleanseMap(properties, replaceUnderScores).foreach { + case (key, Some(value)) => hc.set(key, value.toString) + case (key, None) => hc.unset(key) + case (key, value) => hc.set(key, value.toString) + } + } + + @StepFunction("ea7ea3e0-d1c2-40a2-b2b7-3488489509ca", + "Set Hadoop Configuration Property", + "Set a property on the hadoop configuration.", + "Pipeline", "Spark") + @StepParameters(Map("key" -> StepParameter(None, Some(true), None, None, None, None, Some("The name of the property to set")), + "value" -> StepParameter(None, Some(true), None, None, None, None, Some("The value to set")))) + def setHadoopConfigurationProperty(key: String, value: Any, + pipelineContext: PipelineContext): Unit = { + setHadoopConfigurationProperties(Map(key -> value), Some(false), pipelineContext) + } + + @StepFunction("b7373f02-4d1e-44cf-a9c9-315a5c1ccecc", + "Set Job Group", + "Set the current thread's group id and description that will be associated with any jobs.", + "Pipeline", "Spark") + @StepParameters(Map("groupId" -> StepParameter(None, Some(true), None, None, None, None, Some("The name of the group")), + "description" -> StepParameter(None, Some(true), None, None, None, None, Some("Description of the job group")), + "interruptOnCancel" -> StepParameter(None, Some(false), Some("false"), None, None, None, + Some("When true, will trigger Thread.interrupt getting called on executor threads")))) + def setJobGroup(groupId: String, description: String, interruptOnCancel: Option[Boolean] = None, + pipelineContext: PipelineContext): Unit = { + pipelineContext.sparkSession.get.sparkContext.setJobGroup(groupId, description, interruptOnCancel.getOrElse(false)) + } + + @StepFunction("7394ff4d-f74d-4c9f-a55c-e0fd398fa264", + "Clear Job Group", + "Clear the current thread's job group", + "Pipeline", "Spark") + def clearJobGroup(pipelineContext: PipelineContext): Unit = { + pipelineContext.sparkSession.get.sparkContext.clearJobGroup() + } + + + @tailrec + private def unwrapOptions(value: Any): Any = { + value match { + case Some(v: Option[_]) => unwrapOptions(v) + case v => v + } + } + + private def cleanseMap(map: Map[String, Any], replaceUnderScores: Option[Boolean] = None): Map[String, Any] = { + val replace = replaceUnderScores.getOrElse(true) + map.map{ case (key, value) => + val finalKey = if (replace) key.replaceAllLiterally("_", ".") else key + finalKey -> unwrapOptions(value) + } + } + +} diff --git a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/CSVStepsTests.scala b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/CSVStepsTests.scala index 0e511b0e..f91a0dd9 100644 --- a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/CSVStepsTests.scala +++ b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/CSVStepsTests.scala @@ -12,7 +12,7 @@ import org.scalatest.{BeforeAndAfterAll, FunSpec} class CSVStepsTests extends FunSpec with BeforeAndAfterAll { private val MASTER = "local[2]" - private val APPNAME = "json-steps-spark" + private val APPNAME = "csv-steps-spark" private var sparkConf: SparkConf = _ private var sparkSession: SparkSession = _ private val sparkLocalDir: Path = Files.createTempDirectory("sparkLocal") diff --git a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/SparkConfigurationStepsTests.scala b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/SparkConfigurationStepsTests.scala new file mode 100644 index 00000000..1b1b6350 --- /dev/null +++ b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/SparkConfigurationStepsTests.scala @@ -0,0 +1,118 @@ +package com.acxiom.pipeline.steps + +import java.nio.file.{Files, Path} + +import com.acxiom.pipeline._ +import org.apache.commons.io.FileUtils +import org.apache.log4j.{Level, Logger} +import org.apache.spark.SparkConf +import org.apache.spark.sql.SparkSession +import org.scalatest.{BeforeAndAfterAll, FunSpec, GivenWhenThen} + +class SparkConfigurationStepsTests extends FunSpec with BeforeAndAfterAll with GivenWhenThen { + private val MASTER = "local[2]" + private val APPNAME = "spark-config-steps-spark" + private var sparkConf: SparkConf = _ + private var sparkSession: SparkSession = _ + private val sparkLocalDir: Path = Files.createTempDirectory("sparkLocal") + private var pipelineContext: PipelineContext = _ + + override def beforeAll(): Unit = { + Logger.getLogger("org.apache.spark").setLevel(Level.WARN) + Logger.getLogger("org.apache.hadoop").setLevel(Level.WARN) + Logger.getLogger("com.acxiom.pipeline").setLevel(Level.DEBUG) + + sparkConf = new SparkConf() + .setMaster(MASTER) + .setAppName(APPNAME) + .set("spark.local.dir", sparkLocalDir.toFile.getAbsolutePath) + sparkSession = SparkSession.builder().config(sparkConf).getOrCreate() + + pipelineContext = PipelineContext(Some(sparkConf), Some(sparkSession), Some(Map[String, Any]()), + PipelineSecurityManager(), + PipelineParameters(List(PipelineParameter("0", Map[String, Any]()), PipelineParameter("1", Map[String, Any]()))), + Some(List("com.acxiom.pipeline.steps")), + PipelineStepMapper(), + Some(DefaultPipelineListener()), + Some(sparkSession.sparkContext.collectionAccumulator[PipelineStepMessage]("stepMessages"))) + } + + override def afterAll(): Unit = { + sparkSession.sparkContext.cancelAllJobs() + sparkSession.sparkContext.stop() + sparkSession.stop() + + Logger.getRootLogger.setLevel(Level.INFO) + // cleanup spark directories + FileUtils.deleteDirectory(sparkLocalDir.toFile) + } + + describe("SparkConfigurationSteps - Basic") { + it("should set a local property") { + try { + SparkConfigurationSteps.setLocalProperty("moo", "moo2", pipelineContext) + assert(sparkSession.sparkContext.getLocalProperty("moo") == "moo2") + } finally { + sparkSession.sparkContext.setLocalProperty("moo", None.orNull) + } + } + + it("should unset a local property") { + sparkSession.sparkContext.setLocalProperty("unset", "moo") + SparkConfigurationSteps.setLocalProperty("unset", None, pipelineContext) + assert(Option(sparkSession.sparkContext.getLocalProperty("unset")).isEmpty) + } + + it ("should set a local properties") { + try { + SparkConfigurationSteps.setLocalProperties(Map("moo_m1" -> "m1", "moo_m2" -> "m2"), Some(true), pipelineContext) + assert(sparkSession.sparkContext.getLocalProperty("moo.m1") == "m1") + assert(sparkSession.sparkContext.getLocalProperty("moo.m2") == "m2") + } finally { + sparkSession.sparkContext.setLocalProperty("moo.m1", None.orNull) + sparkSession.sparkContext.setLocalProperty("moo.m2", None.orNull) + } + } + + it ("should unset a local properties") { + try { + sparkSession.sparkContext.setLocalProperty("moo.m1", "m1") + sparkSession.sparkContext.setLocalProperty("moo.m2", "m2") + SparkConfigurationSteps.setLocalProperties(Map("moo_m1" -> None, "moo_m2" -> None), Some(true), pipelineContext) + assert(Option(sparkSession.sparkContext.getLocalProperty("moo.m1")).isEmpty) + assert(Option(sparkSession.sparkContext.getLocalProperty("moo.m2")).isEmpty) + } finally { + sparkSession.sparkContext.setLocalProperty("moo.m1", None.orNull) + sparkSession.sparkContext.setLocalProperty("moo.m2", None.orNull) + } + } + } + + describe("SparkConfigurationSteps - Job Group") { + it("should set a job group") { + SparkConfigurationSteps.setJobGroup("group1", "test1", None, pipelineContext) + val df = sparkSession.range(2) + df.count() + df.head() + val group1Ids = sparkSession.sparkContext.statusTracker.getJobIdsForGroup("group1") + assert(group1Ids.length == 2) + SparkConfigurationSteps.setJobGroup("group2", "test2", None, pipelineContext) + df.count() + val group2Ids = sparkSession.sparkContext.statusTracker.getJobIdsForGroup("group2") + assert(group2Ids.length == 1) + } + + it("should clear a job group") { + SparkConfigurationSteps.setJobGroup("clear1", "test1", None, pipelineContext) + val df = sparkSession.range(2) + df.count() + df.head() + val group1Ids = sparkSession.sparkContext.statusTracker.getJobIdsForGroup("clear1") + assert(group1Ids.length == 2) + SparkConfigurationSteps.clearJobGroup(pipelineContext) + df.count() + assert(sparkSession.sparkContext.statusTracker.getJobIdsForGroup("clear1").length == 2) + } + } + +} From 4ed79baeef3831f465e09a7fa48e7ec03b10afe9 Mon Sep 17 00:00:00 2001 From: djfreels Date: Thu, 14 Oct 2021 11:49:03 -0400 Subject: [PATCH 22/24] tweaking config steps, fixing a bug in S3Utilities. --- .../com/acxiom/aws/utils/S3Utilities.scala | 8 ++++-- .../docs/sparkconfigurationstepds.md | 4 +-- .../steps/SparkConfigurationSteps.scala | 27 +++++++++---------- .../steps/SparkConfigurationStepsTests.scala | 4 +-- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/utils/S3Utilities.scala b/metalus-aws/src/main/scala/com/acxiom/aws/utils/S3Utilities.scala index 2ed4cd78..46fb85f2 100644 --- a/metalus-aws/src/main/scala/com/acxiom/aws/utils/S3Utilities.scala +++ b/metalus-aws/src/main/scala/com/acxiom/aws/utils/S3Utilities.scala @@ -42,17 +42,21 @@ object S3Utilities { role: Option[String] = None, partition: Option[String] = None, pipelineContext: PipelineContext): Unit = { - if (accessKeyId.isDefined && secretAccessKey.isDefined) { + val keyAndSecret = accessKeyId.isDefined && secretAccessKey.isDefined + val roleBased = role.isDefined && accountId.isDefined + if (keyAndSecret || roleBased) { logger.debug(s"Setting up S3 authorization for $path") val protocol = S3Utilities.deriveProtocol(path) val sc = pipelineContext.sparkSession.get.sparkContext if (accessKeyId.isDefined && secretAccessKey.isDefined) { + sc.hadoopConfiguration.unset("spark.hadoop.fs.s3a.aws.credentials.provider") + sc.hadoopConfiguration.unset("fs.s3a.aws.credentials.provider") sc.hadoopConfiguration.set(s"fs.$protocol.awsAccessKeyId", accessKeyId.get) sc.hadoopConfiguration.set(s"fs.$protocol.awsSecretAccessKey", secretAccessKey.get) sc.hadoopConfiguration.set(s"fs.$protocol.access.key", accessKeyId.get) sc.hadoopConfiguration.set(s"fs.$protocol.secret.key", secretAccessKey.get) } - if(role.isDefined && accountId.isDefined && protocol == "s3a") { + if(roleBased && protocol == "s3a") { sc.hadoopConfiguration.set("fs.s3a.assumed.role.arn", buildARN(accountId.get, role.get, partition)) sc.hadoopConfiguration.setStrings("spark.hadoop.fs.s3a.aws.credentials.provider", s"org.apache.hadoop.fs.s3a.AssumedRoleCredentialProvider", diff --git a/metalus-common/docs/sparkconfigurationstepds.md b/metalus-common/docs/sparkconfigurationstepds.md index 800e0687..846a7dd1 100644 --- a/metalus-common/docs/sparkconfigurationstepds.md +++ b/metalus-common/docs/sparkconfigurationstepds.md @@ -13,7 +13,7 @@ Set a local property on the current thread. Set a local property on the current thread for each entry in the properties map. * **properties** - A Map where each entry will be set as a key/value pair. -* **replaceUnderScores** - Replace underscores in the property keys with periods. Enabled by default. +* **keySeparator** - Replaces all occurrences of this string with periods in the keys. Default is __. ##Set Hadoop Configuration Property Set a property on the hadoop configuration. @@ -25,7 +25,7 @@ Set a property on the hadoop configuration. Set a property on the hadoop configuration for each entry in the properties map. * **properties** - A Map where each entry will be set as a key/value pair. -* **replaceUnderScores** - Replace underscores in the property keys with periods. Enabled by default. +* **keySeparator** - Replaces all occurrences of this string with periods in the keys. Default is __. ##Set Job Group Set a job group id and description to group all upcoming jobs on the current thread. diff --git a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/SparkConfigurationSteps.scala b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/SparkConfigurationSteps.scala index 96dce556..0eb86b3f 100644 --- a/metalus-common/src/main/scala/com/acxiom/pipeline/steps/SparkConfigurationSteps.scala +++ b/metalus-common/src/main/scala/com/acxiom/pipeline/steps/SparkConfigurationSteps.scala @@ -15,7 +15,7 @@ object SparkConfigurationSteps { @StepParameters(Map("key" -> StepParameter(None, Some(true), None, None, None, None, Some("The name of the property to set")), "value" -> StepParameter(None, Some(true), None, None, None, None, Some("The value to set")))) def setLocalProperty(key: String, value: Any, pipelineContext: PipelineContext): Unit = { - setLocalProperties(Map(key -> value), Some(false), pipelineContext) + setLocalProperties(Map(key -> value), None, pipelineContext) } @StepFunction("0b86b314-2657-4392-927c-e555af56b415", @@ -24,11 +24,11 @@ object SparkConfigurationSteps { "Pipeline", "Spark") @StepParameters(Map("properties" -> StepParameter(None, Some(true), None, None, None, None, Some("Map representing local properties to set")), - "replaceUnderScores" -> StepParameter(None, Some(false), Some("true"), None, None, None, - Some("When true, will replace underscores in the keys with periods")))) - def setLocalProperties(properties: Map[String, Any], replaceUnderScores: Option[Boolean] = None, pipelineContext: PipelineContext): Unit = { + "keySeparator" -> StepParameter(None, Some(false), Some("__"), None, None, None, + Some("String that will be replaced with a period character")))) + def setLocalProperties(properties: Map[String, Any], keySeparator: Option[String] = None, pipelineContext: PipelineContext): Unit = { val sc = pipelineContext.sparkSession.get.sparkContext - cleanseMap(properties, replaceUnderScores).foreach { + cleanseMap(properties, keySeparator).foreach { case (key, Some(value)) => sc.setLocalProperty(key, value.toString) case (key, None) => sc.setLocalProperty(key, None.orNull) case (key, value) => sc.setLocalProperty(key, value.toString) @@ -41,12 +41,12 @@ object SparkConfigurationSteps { "Pipeline", "Spark") @StepParameters(Map("properties" -> StepParameter(None, Some(true), None, None, None, None, Some("Map representing local properties to set")), - "replaceUnderScores" -> StepParameter(None, Some(false), Some("true"), None, None, None, - Some("When true, will replace underscores in the keys with periods")))) - def setHadoopConfigurationProperties(properties: Map[String, Any], replaceUnderScores: Option[Boolean] = None, + "keySeparator" -> StepParameter(None, Some(false), Some("__"), None, None, None, + Some("String that will be replaced with a period character")))) + def setHadoopConfigurationProperties(properties: Map[String, Any], keySeparator: Option[String] = None, pipelineContext: PipelineContext): Unit = { val hc = pipelineContext.sparkSession.get.sparkContext.hadoopConfiguration - cleanseMap(properties, replaceUnderScores).foreach { + cleanseMap(properties, keySeparator).foreach { case (key, Some(value)) => hc.set(key, value.toString) case (key, None) => hc.unset(key) case (key, value) => hc.set(key, value.toString) @@ -61,7 +61,7 @@ object SparkConfigurationSteps { "value" -> StepParameter(None, Some(true), None, None, None, None, Some("The value to set")))) def setHadoopConfigurationProperty(key: String, value: Any, pipelineContext: PipelineContext): Unit = { - setHadoopConfigurationProperties(Map(key -> value), Some(false), pipelineContext) + setHadoopConfigurationProperties(Map(key -> value), None, pipelineContext) } @StepFunction("b7373f02-4d1e-44cf-a9c9-315a5c1ccecc", @@ -94,11 +94,10 @@ object SparkConfigurationSteps { } } - private def cleanseMap(map: Map[String, Any], replaceUnderScores: Option[Boolean] = None): Map[String, Any] = { - val replace = replaceUnderScores.getOrElse(true) + private def cleanseMap(map: Map[String, Any], keySeparator: Option[String] = None): Map[String, Any] = { + val sep = keySeparator.getOrElse("__") map.map{ case (key, value) => - val finalKey = if (replace) key.replaceAllLiterally("_", ".") else key - finalKey -> unwrapOptions(value) + key.replaceAllLiterally(sep, ".") -> unwrapOptions(value) } } diff --git a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/SparkConfigurationStepsTests.scala b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/SparkConfigurationStepsTests.scala index 1b1b6350..d9922835 100644 --- a/metalus-common/src/test/scala/com/acxiom/pipeline/steps/SparkConfigurationStepsTests.scala +++ b/metalus-common/src/test/scala/com/acxiom/pipeline/steps/SparkConfigurationStepsTests.scala @@ -65,7 +65,7 @@ class SparkConfigurationStepsTests extends FunSpec with BeforeAndAfterAll with G it ("should set a local properties") { try { - SparkConfigurationSteps.setLocalProperties(Map("moo_m1" -> "m1", "moo_m2" -> "m2"), Some(true), pipelineContext) + SparkConfigurationSteps.setLocalProperties(Map("moo_m1" -> "m1", "moo_m2" -> "m2"), Some("_"), pipelineContext) assert(sparkSession.sparkContext.getLocalProperty("moo.m1") == "m1") assert(sparkSession.sparkContext.getLocalProperty("moo.m2") == "m2") } finally { @@ -78,7 +78,7 @@ class SparkConfigurationStepsTests extends FunSpec with BeforeAndAfterAll with G try { sparkSession.sparkContext.setLocalProperty("moo.m1", "m1") sparkSession.sparkContext.setLocalProperty("moo.m2", "m2") - SparkConfigurationSteps.setLocalProperties(Map("moo_m1" -> None, "moo_m2" -> None), Some(true), pipelineContext) + SparkConfigurationSteps.setLocalProperties(Map("moo_m1" -> None, "moo_m2" -> None), Some("_"), pipelineContext) assert(Option(sparkSession.sparkContext.getLocalProperty("moo.m1")).isEmpty) assert(Option(sparkSession.sparkContext.getLocalProperty("moo.m2")).isEmpty) } finally { From 3981366bd7fb89f4af8ee6a76da5a4c728228ffd Mon Sep 17 00:00:00 2001 From: djfreels Date: Thu, 14 Oct 2021 14:49:51 -0400 Subject: [PATCH 23/24] * adding support to limit fork step threads * fixing validation bug that prevented using pipeline mappings for the forkMethod parameter. --- .../PipelineExecutorValidations.scala | 13 ++---- .../acxiom/pipeline/flow/ForkStepFlow.scala | 41 ++++++++++++++++--- .../pipeline/flow/ForkJoinStepTests.scala | 23 +++++++++++ 3 files changed, 62 insertions(+), 15 deletions(-) diff --git a/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineExecutorValidations.scala b/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineExecutorValidations.scala index 35ab76db..26a995c2 100644 --- a/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineExecutorValidations.scala +++ b/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineExecutorValidations.scala @@ -67,21 +67,14 @@ object PipelineExecutorValidations { pipelineProgress = Some(PipelineExecutionInfo(step.id, pipeline.id))) } val forkMethod = step.params.get.find(p => p.name.getOrElse("") == "forkMethod") - if(forkMethod.isDefined && forkMethod.get.value.nonEmpty){ - val method = forkMethod.get.value.get.asInstanceOf[String] - if(!(method == "serial" || method == "parallel")){ - throw PipelineException( - message = Some(s"Unknown value [$method] for parameter [forkMethod]." + - s" Value must be either [serial] or [parallel] for fork step [${step.id.get}] in pipeline [${pipeline.id.get}]."), - pipelineProgress = Some(PipelineExecutionInfo(step.id, pipeline.id))) - } - } else { + + if (forkMethod.flatMap(_.value).isEmpty) { throw PipelineException( message = Some(s"Parameter [forkMethod] is required for fork step [${step.id.get}] in pipeline [${pipeline.id.get}]."), pipelineProgress = Some(PipelineExecutionInfo(step.id, pipeline.id))) } val forkByValues = step.params.get.find(p => p.name.getOrElse("") == "forkByValues") - if(forkByValues.isEmpty || forkByValues.get.value.isEmpty){ + if(forkByValues.flatMap(_.value).isEmpty){ throw PipelineException( message = Some(s"Parameter [forkByValues] is required for fork step [${step.id.get}] in pipeline [${pipeline.id.get}]."), pipelineProgress = Some(PipelineExecutionInfo(step.id, pipeline.id))) diff --git a/metalus-core/src/main/scala/com/acxiom/pipeline/flow/ForkStepFlow.scala b/metalus-core/src/main/scala/com/acxiom/pipeline/flow/ForkStepFlow.scala index 3b7dd458..957d18c7 100644 --- a/metalus-core/src/main/scala/com/acxiom/pipeline/flow/ForkStepFlow.scala +++ b/metalus-core/src/main/scala/com/acxiom/pipeline/flow/ForkStepFlow.scala @@ -1,11 +1,16 @@ package com.acxiom.pipeline.flow import com.acxiom.pipeline._ - import java.util.UUID -import scala.concurrent.ExecutionContext.Implicits.global + import scala.concurrent.duration.Duration -import scala.concurrent.{Await, Future} +import scala.concurrent.forkjoin.ForkJoinPool +import scala.concurrent.{Await, ExecutionContext, Future} +import org.apache.log4j.Logger + +object ForkStepFlow { + val FORK_METHOD_TYPES = List("serial", "parallel") +} case class ForkStepFlow(pipeline: Pipeline, initialContext: PipelineContext, @@ -13,6 +18,9 @@ case class ForkStepFlow(pipeline: Pipeline, executingPipelines: List[Pipeline], step: PipelineStep, parameterValues: Map[String, Any]) extends PipelineFlow { + + private val logger = Logger.getLogger(getClass) + override def execute(): FlowResult = { val result = processForkStep(step, pipeline, stepLookup, parameterValues, initialContext) FlowResult(result.pipelineContext, result.nextStepId, Some(result)) @@ -35,17 +43,31 @@ case class ForkStepFlow(pipeline: Pipeline, val forkFlow = getForkSteps(firstStep, pipeline, steps, ForkFlow(List(), pipeline, List[ForkPair](ForkPair(step, None, root = true)))) forkFlow.validate() val newSteps = forkFlow.steps - // Identify the join steps and verify that only one is present - val joinSteps = newSteps.filter(_.`type`.getOrElse("").toLowerCase == PipelineStepType.JOIN) val newStepLookup = newSteps.foldLeft(Map[String, PipelineStep]())((map, s) => map + (s.id.get -> s)) // See if the forks should be executed in threads or a loop val forkByValues = parameterValues("forkByValues").asInstanceOf[List[Any]] + val forkMethod = parameterValues("forkMethod") + if (!ForkStepFlow.FORK_METHOD_TYPES.contains(forkMethod.toString.toLowerCase)) { + throw PipelineException( + message = Some(s"Unsupported value [$forkMethod] for parameter [forkMethod]." + + s" Value must be one of the supported values [${ForkStepFlow.FORK_METHOD_TYPES.mkString(", ")}]" + + s" for fork step [${step.id.get}] in pipeline [${pipeline.id.get}]."), + pipelineProgress = Some(pipelineContext.getPipelineExecutionInfo)) + } val results = if (parameterValues("forkMethod").asInstanceOf[String] == "parallel") { processForkStepsParallel(forkByValues, firstStep, step.id.get, pipeline, newStepLookup, pipelineContext) } else { // "serial" processForkStepsSerial(forkByValues, firstStep, step.id.get, pipeline, newStepLookup, pipelineContext) } // Gather the results and create a list + handleResults(results, forkFlow, steps, pipelineContext) + } + + private def handleResults(results: List[ForkStepExecutionResult], forkFlow: ForkFlow, steps: Map[String, PipelineStep], + pipelineContext: PipelineContext): ForkStepResult = { + val newSteps = forkFlow.steps + // Identify the join steps and verify that only one is present + val joinSteps = newSteps.filter(_.`type`.getOrElse("").toLowerCase == PipelineStepType.JOIN) val finalResult = results.sortBy(_.index).foldLeft(ForkStepExecutionResult(-1, Some(pipelineContext), None))((combinedResult, result) => { if (result.result.isDefined) { val ctx = result.result.get @@ -233,6 +255,15 @@ case class ForkStepFlow(pipeline: Pipeline, pipeline: Pipeline, steps: Map[String, PipelineStep], pipelineContext: PipelineContext): List[ForkStepExecutionResult] = { + implicit val executionContext: ExecutionContext = parameterValues.get("forkLimit").flatMap{ v => + val limit = v.toString.trim + if (limit.forall(_.isDigit)) { + Some(ExecutionContext.fromExecutorService(new ForkJoinPool(limit.toInt))) + } else { + logger.warn(s"Unable to parse forkLimit value: [$limit] as integer. Defaulting to global ExecutionContext.") + None + } + }.getOrElse(scala.concurrent.ExecutionContext.Implicits.global) val futures = forkByValues.zipWithIndex.map(value => { Future { startForkedStepExecution(firstStep, forkStepId, pipeline, steps, diff --git a/metalus-core/src/test/scala/com/acxiom/pipeline/flow/ForkJoinStepTests.scala b/metalus-core/src/test/scala/com/acxiom/pipeline/flow/ForkJoinStepTests.scala index 0e3ca1fd..aa96c535 100644 --- a/metalus-core/src/test/scala/com/acxiom/pipeline/flow/ForkJoinStepTests.scala +++ b/metalus-core/src/test/scala/com/acxiom/pipeline/flow/ForkJoinStepTests.scala @@ -75,6 +75,11 @@ class ForkJoinStepTests extends FunSpec with BeforeAndAfterAll with Suite { private val errorValueStep = PipelineStep(Some("EXCEPTION"), None, None, Some("Pipeline"), Some(List(Parameter(Some("text"), Some("string"), value = Some("@FORK_DATA")))), engineMeta = Some(EngineMeta(Some("MockStepObject.mockExceptionStepFunction")))) + private val simpleForkParallelStepWithLimit = PipelineStep(Some("FORK_DATA"), None, None, Some("fork"), + Some(List(Parameter(Some("text"), Some("forkByValues"), value = Some("@GENERATE_DATA")), + Parameter(Some("text"), Some("forkMethod"), value = Some("parallel")), + Parameter(Some("text"), Some("forkLimit"), value = Some("2")))), + nextStepId = Some("PROCESS_VALUE")) describe("Fork Step Without Join") { it("Should process list and merge results using serial processing") { @@ -91,6 +96,14 @@ class ForkJoinStepTests extends FunSpec with BeforeAndAfterAll with Suite { val executionResult = PipelineExecutor.executePipelines(List(pipeline), Some("PARALLEL_FORK_TEST"), SparkTestHelper.generatePipelineContext()) verifySimpleForkSteps(pipeline, executionResult) } + + it("Should process list and merge results using parallel processing with limit") { + val pipelineSteps = List(generateDataStep, simpleForkParallelStepWithLimit, processValueStep) + val pipeline = Pipeline(Some("PARALLEL_FORK_TEST"), Some("Parallel Fork Test"), Some(pipelineSteps)) + SparkTestHelper.pipelineListener = PipelineListener() + val executionResult = PipelineExecutor.executePipelines(List(pipeline), Some("PARALLEL_FORK_TEST"), SparkTestHelper.generatePipelineContext()) + verifySimpleForkSteps(pipeline, executionResult) + } } describe("Fork Step With Join") { @@ -112,6 +125,16 @@ class ForkJoinStepTests extends FunSpec with BeforeAndAfterAll with Suite { val executionResult = PipelineExecutor.executePipelines(List(pipeline), None, SparkTestHelper.generatePipelineContext()) verifySimpleForkSteps(pipeline, executionResult, extraStep = true) } + + it("Should process list and merge results using parallel processing with limit") { + val pipeline = Pipeline(Some("PARALLEL_FORK_TEST"), Some("Parallel Fork Test"), Some( + List(generateDataStep, simpleForkParallelStepWithLimit, processValueStep.copy(nextStepId = Some("BRANCH_VALUE")), + simpleBranchStep, joinStep.copy(nextStepId = Some("PROCESS_RAW_VALUE")), simpleMockStep) + )) + SparkTestHelper.pipelineListener = PipelineListener() + val executionResult = PipelineExecutor.executePipelines(List(pipeline), None, SparkTestHelper.generatePipelineContext()) + verifySimpleForkSteps(pipeline, executionResult, extraStep = true) + } } describe("Embedded fork steps") { From eeab70fc6f3b583524047dfb16727d211213e537 Mon Sep 17 00:00:00 2001 From: djfreels Date: Fri, 15 Oct 2021 13:10:42 -0400 Subject: [PATCH 24/24] Added support to map from rootGlobals to application globals --- .../testData/metalus-common/steps.json | 2 +- .../acxiom/pipeline/PipelineStepMapper.scala | 7 +-- .../applications/ApplicationUtils.scala | 60 ++++++++++++------- .../applications/ApplicationTests.scala | 8 +-- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/manual_tests/testData/metalus-common/steps.json b/manual_tests/testData/metalus-common/steps.json index f400fd4f..589a9b2e 100644 --- a/manual_tests/testData/metalus-common/steps.json +++ b/manual_tests/testData/metalus-common/steps.json @@ -1 +1 @@ -{"pkgs":["com.acxiom.pipeline.steps"],"steps":[{"id":"3806f23b-478c-4054-b6c1-37f11db58d38","displayName":"Read a DataFrame from Table","description":"This step will read a dataFrame in a given format from the meta store","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"table","required":true,"parameterType":"String","description":"The name of the table to read"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The DataFrameReaderOptions to use"}],"engineMeta":{"spark":"CatalogSteps.readDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"e2b4c011-e71b-46f9-a8be-cf937abc2ec4","displayName":"Write DataFrame to Table","description":"This step will write a dataFrame in a given format to the meta store","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to write"},{"type":"text","name":"table","required":true,"parameterType":"String","description":"The name of the table to write to"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The DataFrameWriterOptions to use"}],"engineMeta":{"spark":"CatalogSteps.writeDataFrame","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"5874ab64-13c7-404c-8a4f-67ff3b0bc7cf","displayName":"Drop Catalog Object","description":"This step will drop an object from the meta store","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the object to drop"},{"type":"text","name":"objectType","required":false,"defaultValue":"TABLE","parameterType":"String","description":"Type of object to drop"},{"type":"boolean","name":"ifExists","required":false,"defaultValue":"false","parameterType":"Boolean","description":"Flag to control whether existence is checked"},{"type":"boolean","name":"cascade","required":false,"defaultValue":"false","parameterType":"Boolean","description":"Flag to control whether this deletion should cascade"}],"engineMeta":{"spark":"CatalogSteps.drop","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"17be71f9-1492-4404-a355-1cc973694cad","displayName":"Database Exists","description":"Check spark catalog for a database with the given name.","type":"branch","category":"Decision","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the database"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"CatalogSteps.databaseExists","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"95181811-d83e-4136-bedb-2cba1de90301","displayName":"Table Exists","description":"Check spark catalog for a table with the given name.","type":"branch","category":"Decision","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the table"},{"type":"text","name":"database","required":false,"parameterType":"String","description":"Name of the database"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"CatalogSteps.tableExists","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f4adfe70-2ae3-4b8d-85d1-f53e91c8dfad","displayName":"Set Current Database","description":"Set the current default database for the spark session.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the database"}],"engineMeta":{"spark":"CatalogSteps.setCurrentDatabase","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"663f8c93-0a42-4c43-8263-33f89c498760","displayName":"Create Table","description":"Create a table in the meta store.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the table"},{"type":"text","name":"externalPath","required":false,"parameterType":"String","description":"Path of the external table"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"Options containing the format, schema, and settings"}],"engineMeta":{"spark":"CatalogSteps.createTable","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"836aab38-1140-4606-ab73-5b6744f0e7e7","displayName":"Load","description":"This step will create a DataFrame using the given DataConnector","type":"Pipeline","category":"Connectors","params":[{"type":"text","name":"connector","required":true,"parameterType":"com.acxiom.pipeline.connectors.DataConnector","description":"The data connector to use when writing"},{"type":"text","name":"source","required":false,"parameterType":"String","description":"The source path to load data"},{"type":"object","name":"readOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The optional options to use while reading the data"}],"engineMeta":{"spark":"DataConnectorSteps.loadDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"5608eba7-e9ff-48e6-af77-b5e810b99d89","displayName":"Write","description":"This step will write a DataFrame using the given DataConnector","type":"Pipeline","category":"Connectors","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.DataFrame","description":"The DataFrame to write"},{"type":"text","name":"connector","required":true,"parameterType":"com.acxiom.pipeline.connectors.DataConnector","description":"The data connector to use when writing"},{"type":"text","name":"destination","required":false,"parameterType":"String","description":"The destination path to write data"},{"type":"object","name":"writeOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The optional DataFrame options to use while writing"}],"engineMeta":{"spark":"DataConnectorSteps.writeDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.streaming.StreamingQuery"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"87db259d-606e-46eb-b723-82923349640f","displayName":"Load DataFrame from HDFS path","description":"This step will read a dataFrame from the given HDFS path","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"path","required":true,"parameterType":"String","description":"The HDFS path to load data into the DataFrame"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"HDFSSteps.readFromPath","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"8daea683-ecde-44ce-988e-41630d251cb8","displayName":"Load DataFrame from HDFS paths","description":"This step will read a dataFrame from the given HDFS paths","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"paths","required":true,"parameterType":"List[String]","description":"The HDFS paths to load data into the DataFrame"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"HDFSSteps.readFromPaths","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"0a296858-e8b7-43dd-9f55-88d00a7cd8fa","displayName":"Write DataFrame to HDFS","description":"This step will write a dataFrame in a given format to HDFS","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to write"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The GCS path to write data"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The optional DataFrame Options"}],"engineMeta":{"spark":"HDFSSteps.writeToPath","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"e4dad367-a506-5afd-86c0-82c2cf5cd15c","displayName":"Create HDFS FileManager","description":"Simple function to generate the HDFSFileManager for the local HDFS file system","type":"Pipeline","category":"InputOutput","params":[],"engineMeta":{"spark":"HDFSSteps.createFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.HDFSFileManager"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"a7e17c9d-6956-4be0-a602-5b5db4d1c08b","displayName":"Scala script Step","description":"Executes a script and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"scala","parameterType":"String","description":"A scala script to execute"}],"engineMeta":{"spark":"ScalaSteps.processScript","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"8bf8cef6-cf32-4d85-99f4-e4687a142f84","displayName":"Scala script Step with additional object provided","description":"Executes a script with the provided object and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"scala","parameterType":"String","description":"A scala script to execute"},{"type":"text","name":"value","required":true,"parameterType":"Any","description":"A value to pass to the script"},{"type":"text","name":"type","required":false,"parameterType":"String","description":"The type of the value to pass to the script"}],"engineMeta":{"spark":"ScalaSteps.processScriptWithValue","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3ab721e8-0075-4418-aef1-26abdf3041be","displayName":"Scala script Step with additional objects provided","description":"Executes a script with the provided object and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"scala","parameterType":"String","description":"A scala script to execute"},{"type":"object","name":"values","required":true,"parameterType":"Map[String,Any]","description":"Map of name/value pairs that will be bound to the script"},{"type":"object","name":"types","required":false,"parameterType":"Map[String,String]","description":"Map of type overrides for the values provided"},{"type":"boolean","name":"unwrapOptions","required":false,"parameterType":"Boolean","description":"Flag to toggle option unwrapping behavior"}],"engineMeta":{"spark":"ScalaSteps.processScriptWithValues","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"6e42b0c3-340e-4848-864c-e1b5c57faa4f","displayName":"Join DataFrames","description":"Join two dataFrames together.","type":"Pipeline","category":"Data","params":[{"type":"text","name":"left","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"Left side of the join"},{"type":"text","name":"right","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"Right side of the join"},{"type":"text","name":"expression","required":false,"parameterType":"String","description":"Join expression. Optional for cross joins"},{"type":"text","name":"leftAlias","required":false,"defaultValue":"left","parameterType":"String","description":"Left side alias"},{"type":"text","name":"rightAlias","required":false,"defaultValue":"right","parameterType":"String","description":"Right side alias"},{"type":"text","name":"joinType","required":false,"defaultValue":"inner","parameterType":"String","description":"Type of join to perform"}],"engineMeta":{"spark":"DataSteps.join","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"823eeb28-ec81-4da6-83f2-24a1e580b0e5","displayName":"Group By","description":"Group by a list of grouping expressions and a list of aggregates.","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to group"},{"type":"text","name":"groupings","required":true,"parameterType":"List[String]","description":"List of expressions to group by"},{"type":"text","name":"aggregations","required":true,"parameterType":"List[String]","description":"List of aggregations to apply"}],"engineMeta":{"spark":"DataSteps.groupBy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"d322769c-18a0-49c2-9875-41446892e733","displayName":"Union","description":"Union two DataFrames together.","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The initial DataFrame"},{"type":"text","name":"append","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The dataFrame to append"},{"type":"boolean","name":"distinct","required":false,"defaultValue":"true","parameterType":"Boolean","description":"Flag to control distinct behavior"}],"engineMeta":{"spark":"DataSteps.union","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"80583aa9-41b7-4906-8357-cc2d3670d970","displayName":"Add a Column with a Static Value to All Rows in a DataFrame (metalus-common)","description":"This step will add a column with a static value to all rows in the provided data frame","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The data frame to add the column"},{"type":"text","name":"columnName","required":true,"parameterType":"String","description":"The name to provide the id column"},{"type":"text","name":"columnValue","required":true,"parameterType":"Any","description":"The name of the new column"},{"type":"boolean","name":"standardizeColumnName","required":false,"defaultValue":"true","parameterType":"Boolean","description":"The value to add"}],"engineMeta":{"spark":"DataSteps.addStaticColumnToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"e625eed6-51f0-44e7-870b-91c960cdc93d","displayName":"Adds a Unique Identifier to a DataFrame (metalus-common)","description":"This step will add a new unique identifier to an existing data frame using the monotonically_increasing_id method","type":"Pipeline","category":"Data","params":[{"type":"text","name":"idColumnName","required":true,"parameterType":"String","description":"The name to provide the id column"},{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The data frame to add the column"}],"engineMeta":{"spark":"DataSteps.addUniqueIdToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"fa0fcabb-d000-4a5e-9144-692bca618ddb","displayName":"Filter a DataFrame","description":"This step will filter a DataFrame based on the where expression provided","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to filter"},{"type":"text","name":"expression","required":true,"parameterType":"String","description":"The expression to apply to the DataFrame to filter rows"}],"engineMeta":{"spark":"DataSteps.applyFilter","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"5d0d7c5c-c287-4565-80b2-2b1a847b18c6","displayName":"Get DataFrame Count","description":"Get a count of records in a DataFrame.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to count"}],"engineMeta":{"spark":"DataSteps.getDataFrameCount","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Long"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"252b6086-da45-4042-a9a8-31ebf57948af","displayName":"Drop Duplicate Records","description":"Drop duplicate records from a DataFrame","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to drop duplicate records from"},{"type":"text","name":"columnNames","required":true,"parameterType":"List[String]","description":"Columns to use for determining distinct values to drop"}],"engineMeta":{"spark":"DataSteps.dropDuplicateRecords","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"d5ac88a2-caa2-473c-a9f7-ffb0269880b2","displayName":"Rename Column","description":"Rename a column on a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to change"},{"type":"text","name":"oldColumnName","required":true,"parameterType":"String","description":"The name of the column you want to change"},{"type":"text","name":"newColumnName","required":true,"parameterType":"String","description":"The new name to give the column"}],"engineMeta":{"spark":"DataSteps.renameColumn","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"6ed36f89-35d1-4280-a555-fbcd8dd76bf2","displayName":"Retry (simple)","description":"Makes a decision to retry or stop based on a named counter","type":"branch","category":"RetryLogic","params":[{"type":"text","name":"counterName","required":true,"parameterType":"String","description":"The name of the counter to use for tracking"},{"type":"text","name":"maxRetries","required":true,"parameterType":"Int","description":"The maximum number of retries allowed"},{"type":"result","name":"retry","required":false},{"type":"result","name":"stop","required":false}],"engineMeta":{"spark":"FlowUtilsSteps.simpleRetry","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"a2f3e151-cb81-4c69-8475-c1a287bbb4cb","displayName":"Convert CSV String Dataset to DataFrame","description":"This step will convert the provided CSV string Dataset into a DataFrame that can be passed to other steps","type":"Pipeline","category":"CSV","params":[{"type":"text","name":"dataset","required":true,"parameterType":"org.apache.spark.sql.Dataset[String]","description":"The dataset containing CSV strings"},{"type":"object","name":"dataFrameReaderOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The CSV parsing options"}],"engineMeta":{"spark":"CSVSteps.csvDatasetToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"d25209c1-53f6-49ad-a402-257ae756ac2a","displayName":"Convert CSV String to DataFrame","description":"This step will convert the provided CSV string into a DataFrame that can be passed to other steps","type":"Pipeline","category":"CSV","params":[{"type":"text","name":"csvString","required":true,"parameterType":"String","description":"The csv string to convert to a DataFrame"},{"type":"text","name":"delimiter","required":false,"defaultValue":",","parameterType":"String","description":"The field delimiter"},{"type":"text","name":"recordDelimiter","required":false,"defaultValue":"\\n","parameterType":"String","description":"The record delimiter"},{"type":"boolean","name":"header","required":false,"defaultValue":"false","parameterType":"Boolean","description":"Build header from the first row"}],"engineMeta":{"spark":"CSVSteps.csvStringToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"15889487-fd1c-4c44-b8eb-973c12f91fae","displayName":"Creates an HttpRestClient","description":"This step will build an HttpRestClient using a host url and optional authorization object","type":"Pipeline","category":"API","params":[{"type":"text","name":"hostUrl","required":true,"parameterType":"String","description":"The URL to connect including port"},{"type":"text","name":"authorization","required":false,"parameterType":"com.acxiom.pipeline.api.Authorization","description":"The optional authorization class to use when making connections"},{"type":"boolean","name":"allowSelfSignedCertificates","required":false,"parameterType":"Boolean","description":"Flag to allow using self signed certificates for http calls"}],"engineMeta":{"spark":"ApiSteps.createHttpRestClient","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.api.HttpRestClient"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"fcfd4b91-9a9c-438c-8afa-9f14c1e52a82","displayName":"Creates an HttpRestClient from protocol, host and port","description":"This step will build an HttpRestClient using url parts and optional authorization object","type":"Pipeline","category":"API","params":[{"type":"text","name":"protocol","required":true,"parameterType":"String","description":"The protocol to use when constructing the URL"},{"type":"text","name":"host","required":true,"parameterType":"String","description":"The host name to use when constructing the URL"},{"type":"text","name":"port","required":true,"parameterType":"Int","description":"The port to use when constructing the URL"},{"type":"text","name":"authorization","required":false,"parameterType":"com.acxiom.pipeline.api.Authorization","description":"The optional authorization class to use when making connections"}],"engineMeta":{"spark":"ApiSteps.createHttpRestClientFromParameters","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.api.HttpRestClient"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"b59f0486-78aa-4bd4-baf5-5c7d7c648ff0","displayName":"Check Path Exists","description":"Checks the path to determine whether it exists or not.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to verify"}],"engineMeta":{"spark":"ApiSteps.exists","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"7521ac47-84ec-4e50-b087-b9de4bf6d514","displayName":"Get the last modified date","description":"Gets the last modified date for the provided path","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource to get the last modified date"}],"engineMeta":{"spark":"ApiSteps.getLastModifiedDate","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.util.Date"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"fff7f7b6-5d9a-40b3-8add-6432552920a8","displayName":"Get Path Content Length","description":"Get the size of the content at the given path.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource to get the content length"}],"engineMeta":{"spark":"ApiSteps.getContentLength","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Long"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"dd351d47-125d-47fa-bafd-203bebad82eb","displayName":"Get Path Headers","description":"Get the headers for the content at the given path.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to get the headers"}],"engineMeta":{"spark":"ApiSteps.getHeaders","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Map[String,List[String]]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"532f72dd-8443-481d-8406-b74cdc08e342","displayName":"Delete Content","description":"Attempts to delete the provided path..","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to delete"}],"engineMeta":{"spark":"ApiSteps.delete","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3b91e6e8-ec18-4468-9089-8474f4b4ba48","displayName":"GET String Content","description":"Retrieves the value at the provided path as a string.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to resource"}],"engineMeta":{"spark":"ApiSteps.getStringContent","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"34c2fc9a-2502-4c79-a0cb-3f866a0a0d6e","displayName":"POST String Content","description":"POSTs the provided string to the provided path using the content type and returns the response as a string.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to post the content"},{"type":"text","name":"content","required":true,"parameterType":"String","description":"The content to post"},{"type":"text","name":"contentType","required":false,"parameterType":"String","description":"The content type being sent to the path"}],"engineMeta":{"spark":"ApiSteps.postStringContent","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"49ae38b3-cb41-4153-9111-aa6aacf6721d","displayName":"PUT String Content","description":"PUTs the provided string to the provided path using the content type and returns the response as a string.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to post the content"},{"type":"text","name":"content","required":true,"parameterType":"String","description":"The content to put"},{"type":"text","name":"contentType","required":false,"parameterType":"String","description":"The content type being sent to the path"}],"engineMeta":{"spark":"ApiSteps.putStringContent","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"99b20c23-722f-4862-9f47-bc9f72440ae6","displayName":"GET Input Stream","description":"Creates a buffered input stream for the provided path","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource"},{"type":"text","name":"bufferSize","required":false,"parameterType":"Int","description":"The size of buffer to use with the stream"}],"engineMeta":{"spark":"ApiSteps.getInputStream","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.io.InputStream"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f4120b1c-91df-452f-9589-b77f8555ba44","displayName":"GET Output Stream","description":"Creates a buffered output stream for the provided path.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource"},{"type":"text","name":"bufferSize","required":false,"parameterType":"Int","description":"The size of buffer to use with the stream"}],"engineMeta":{"spark":"ApiSteps.getOutputStream","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.io.OutputStream"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"cdb332e3-9ea4-4c96-8b29-c1d74287656c","displayName":"Load table as DataFrame using JDBCOptions","description":"This step will load a table from the provided JDBCOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"jdbcOptions","required":true,"parameterType":"org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions","description":"The options to use when loading the DataFrame"}],"engineMeta":{"spark":"JDBCSteps.readWithJDBCOptions","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"72dbbfc8-bd1d-4ce4-ab35-28fa8385ea54","displayName":"Load table as DataFrame using StepOptions","description":"This step will load a table from the provided JDBCDataFrameReaderOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"object","name":"jDBCStepsOptions","required":true,"className":"com.acxiom.pipeline.steps.JDBCDataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.JDBCDataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"JDBCSteps.readWithStepOptions","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"dcc57409-eb91-48c0-975b-ca109ba30195","displayName":"Load table as DataFrame","description":"This step will load a table from the provided jdbc information","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"url","required":true,"parameterType":"String","description":"A valid jdbc url"},{"type":"text","name":"table","required":true,"parameterType":"String","description":"A table name or subquery"},{"type":"text","name":"predicates","required":false,"parameterType":"List[String]","description":"Optional predicates used for partitioning"},{"type":"text","name":"connectionProperties","required":false,"parameterType":"Map[String,String]","description":"Optional properties for the jdbc connection"}],"engineMeta":{"spark":"JDBCSteps.readWithProperties","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"c9fddf52-34b1-4216-a049-10c33ccd24ab","displayName":"Write DataFrame to table using JDBCOptions","description":"This step will write a DataFrame as a table using JDBCOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to be written"},{"type":"text","name":"jdbcOptions","required":true,"parameterType":"org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions","description":"Options for configuring the JDBC connection"},{"type":"text","name":"saveMode","required":false,"parameterType":"String","description":"The value for the mode option. Defaulted to Overwrite"}],"engineMeta":{"spark":"JDBCSteps.writeWithJDBCOptions","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"77ffcd02-fbd0-4f79-9b35-ac9dc5fb7190","displayName":"Write DataFrame to table","description":"This step will write a DataFrame to a table using the provided properties","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to be written"},{"type":"text","name":"url","required":true,"parameterType":"String","description":"A valid jdbc url"},{"type":"text","name":"table","required":true,"parameterType":"String","description":"A table name or subquery"},{"type":"text","name":"connectionProperties","required":false,"parameterType":"Map[String,String]","description":"Optional properties for the jdbc connection"},{"type":"text","name":"saveMode","required":false,"parameterType":"String","description":"The value for the mode option. Defaulted to Overwrite"}],"engineMeta":{"spark":"JDBCSteps.writeWithProperties","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3d6b77a1-52c2-49ba-99a0-7ec773dac696","displayName":"Write DataFrame to JDBC table","description":"This step will write a DataFrame to a table using the provided JDBCDataFrameWriterOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to be written"},{"type":"object","name":"jDBCStepsOptions","required":true,"className":"com.acxiom.pipeline.steps.JDBCDataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.JDBCDataFrameWriterOptions","description":"Options for the JDBC connect and spark DataFrameWriter"}],"engineMeta":{"spark":"JDBCSteps.writeWithStepOptions","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"713fff3d-d407-4970-89ae-7844e6fc60e3","displayName":"Get JDBC Connection","description":"Get a jdbc connection.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"url","required":true,"parameterType":"String","description":"A valid jdbc url"},{"type":"text","name":"properties","required":false,"parameterType":"Map[String,String]","description":"Optional properties for the jdbc connection"}],"engineMeta":{"spark":"JDBCSteps.getConnection","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.sql.Connection"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"549828be-3d96-4561-bf94-7ad420f9d203","displayName":"Execute Sql","description":"Execute a sql command using jdbc.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"sql","required":true,"parameterType":"String","description":"Sql command to execute"},{"type":"text","name":"connection","required":true,"parameterType":"java.sql.Connection","description":"An open jdbc connection"},{"type":"text","name":"parameters","required":false,"parameterType":"List[Any]","description":"Optional list of bind variables"}],"engineMeta":{"spark":"JDBCSteps.executeSql","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"9c8957a3-899e-4f32-830e-d120b1917aa1","displayName":"Close JDBC Connection","description":"Close a JDBC Connection.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"connection","required":true,"parameterType":"java.sql.Connection","description":"An open jdbc connection"}],"engineMeta":{"spark":"JDBCSteps.closeConnection","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3464dc85-5111-40fc-9bfb-1fd6fc8a2c17","displayName":"Convert JSON String to Map","description":"This step will convert the provided JSON string into a Map that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonString","required":true,"parameterType":"String","description":"The JSON string to convert to a map"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonStringToMap","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Map[String,Any]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f4d19691-779b-4962-a52b-ee5d9a99068e","displayName":"Convert JSON Map to JSON String","description":"This step will convert the provided JSON map into a JSON string that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonMap","required":true,"parameterType":"Map[String,Any]","description":"The JSON map to convert to a JSON string"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonMapToString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"1f23eb37-98ee-43c2-ac78-17b04db3cc8d","displayName":"Convert object to JSON String","description":"This step will convert the provided object into a JSON string that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"obj","required":true,"parameterType":"AnyRef","description":"The object to convert to a JSON string"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.objectToJsonString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"880c5151-f7cd-40bb-99f2-06dbb20a6523","displayName":"Convert JSON String to object","description":"This step will convert the provided JSON string into an object that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonString","required":true,"parameterType":"String","description":"The JSON string to convert to an object"},{"type":"text","name":"objectName","required":true,"parameterType":"String","description":"The fully qualified class name of the object"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonStringToObject","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Any"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"68958a29-aab5-4f7e-9ffd-af99c33c512b","displayName":"Convert JSON String to Schema","description":"This step will convert the provided JSON string into a Schema that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"schema","required":true,"parameterType":"String","description":"The JSON string to convert to a Schema"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonStringToSchema","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.Schema"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"cf4e9e6c-98d6-4a14-ae74-52322782c504","displayName":"Convert JSON String to DataFrame","description":"This step will convert the provided JSON string into a DataFrame that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonString","required":true,"parameterType":"String","description":"The JSON string to convert to a DataFrame"}],"engineMeta":{"spark":"JSONSteps.jsonStringToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"d5cd835e-5e8f-49c0-9706-746d5a4d7b3a","displayName":"Convert JSON String Dataset to DataFrame","description":"This step will convert the provided JSON string Dataset into a DataFrame that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"dataset","required":true,"parameterType":"org.apache.spark.sql.Dataset[String]","description":"The dataset containing JSON strings"},{"type":"object","name":"dataFrameReaderOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The JSON parsing options"}],"engineMeta":{"spark":"JSONSteps.jsonDatasetToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f3891201-5138-4cab-aebc-bcc319228543","displayName":"Build JSON4S Formats","description":"This step will build a json4s Formats object that can be used to override the default","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"customSerializers","required":false,"parameterType":"List[com.acxiom.pipeline.applications.ClassInfo]","description":"List of custom serializer classes"},{"type":"text","name":"enumIdSerializers","required":false,"parameterType":"List[com.acxiom.pipeline.applications.ClassInfo]","description":"List of Enumeration classes to serialize by id"},{"type":"text","name":"enumNameSerializers","required":false,"parameterType":"List[com.acxiom.pipeline.applications.ClassInfo]","description":"List of Enumeration classes to serialize by name"}],"engineMeta":{"spark":"JSONSteps.buildJsonFormats","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.json4s.Formats"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"b5485d97-d4e8-41a6-8af7-9ce79a435140","displayName":"To String","description":"Returns the result of the toString method, can unwrap options","type":"Pipeline","category":"String","params":[{"type":"text","name":"value","required":true,"parameterType":"Any","description":"The value to convert"},{"type":"boolean","name":"unwrapOption","required":false,"parameterType":"Boolean","description":"Boolean indicating whether to unwrap the value from an Option prior to calling toString"}],"engineMeta":{"spark":"StringSteps.toString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"78e817ec-2bf2-4cbe-acba-e5bc9bdcffc5","displayName":"List To String","description":"Returns the result of the mkString method","type":"Pipeline","category":"String","params":[{"type":"text","name":"list","required":true,"parameterType":"List[Any]","description":"The list to convert"},{"type":"text","name":"separator","required":false,"parameterType":"String","description":"Separator character to use when making the string"},{"type":"boolean","name":"unwrapOptions","required":false,"parameterType":"Boolean","description":"Boolean indicating whether to unwrap each value from an Option"}],"engineMeta":{"spark":"StringSteps.listToString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"fcd6b5fe-08ed-4cfd-acfe-eb676d7f4ecd","displayName":"To Lowercase","description":"Returns a lowercase string","type":"Pipeline","category":"String","params":[{"type":"text","name":"value","required":true,"parameterType":"String","description":"The value to lowercase"}],"engineMeta":{"spark":"StringSteps.toLowerCase","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"2f31ebf1-4ae2-4e04-9b29-4802cac8a198","displayName":"To Uppercase","description":"Returns an uppercase string","type":"Pipeline","category":"String","params":[{"type":"text","name":"value","required":true,"parameterType":"String","description":"The value to uppercase"}],"engineMeta":{"spark":"StringSteps.toUpperCase","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"96b7b521-5304-4e63-8435-63d84a358368","displayName":"String Split","description":"Returns a list of strings split off of the given string","type":"Pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to split"},{"type":"text","name":"regex","required":true,"parameterType":"String","description":"Regex to use when splitting the string"},{"type":"integer","name":"limit","required":false,"parameterType":"Int","description":"Max number elements to return in the list"}],"engineMeta":{"spark":"StringSteps.stringSplit","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"List[String]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f75abedd-4aee-4979-8d56-ea7b0c1a86e1","displayName":"Substring","description":"Returns a substring","type":"Pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to parse"},{"type":"text","name":"begin","required":true,"parameterType":"Int","description":"The beginning index"},{"type":"integer","name":"end","required":false,"parameterType":"Int","description":"The end index"}],"engineMeta":{"spark":"StringSteps.substring","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3fabf9ec-5383-4eb3-81af-6092ab7c370d","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","category":"Decision","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"parameterType":"String","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"parameterType":"Boolean","description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"ff0562f5-2917-406d-aa78-c5d49ba6b99f","displayName":"String Matches","description":"Return whether string matches a given regex","type":"branch","category":"Decision","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to match"},{"type":"text","name":"regex","required":true,"parameterType":"String","description":"Regex to use for the match"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"StringSteps.stringMatches","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"416baf4e-a1dd-49fc-83a9-0f41b77e57b7","displayName":"String Replace All","description":"Perform a literal or regex replacement on a string","type":"pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to modify"},{"type":"text","name":"matchString","required":true,"parameterType":"String","description":"The string to match"},{"type":"text","name":"replacement","required":false,"parameterType":"String","description":"The replacement string"},{"type":"boolean","name":"literal","required":false,"parameterType":"Boolean","description":"Perform \\'literal\\' match replacement"}],"engineMeta":{"spark":"StringSteps.stringReplaceAll","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"95438b82-8d50-41da-8094-c92449b9e7df","displayName":"String Replace First","description":"Perform a literal or regex replacement on the first occurrence in a string","type":"pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to modify"},{"type":"text","name":"matchString","required":true,"parameterType":"String","description":"The string to match"},{"type":"text","name":"replacement","required":false,"parameterType":"String","description":"The replacement string"},{"type":"boolean","name":"literal","required":false,"parameterType":"Boolean","description":"Perform \\'literal\\' match replacement"}],"engineMeta":{"spark":"StringSteps.stringReplaceFirst","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"86c84fa3-ad45-4a49-ac05-92385b8e9572","displayName":"Get Credential","description":"This step provides access to credentials through the CredentialProvider","type":"Pipeline","category":"Credentials","params":[{"type":"text","name":"credentialName","required":true,"parameterType":"String","description":"The dataset containing CSV strings"}],"engineMeta":{"spark":"CredentialSteps.getCredential","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.Credential"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"219c787a-f502-4efc-b15d-5beeff661fc0","displayName":"Map a DataFrame to an existing DataFrame","description":"This step maps a new DataFrame to an existing DataFrame to make them compatible","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"inputDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame that needs to be modified"},{"type":"text","name":"destinationDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame that the new data needs to map to"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations","description":"The object with transform, alias, and filter logic details"},{"type":"boolean","name":"addNewColumns","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.mapToDestinationDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"8f9c08ea-4882-4265-bac7-2da3e942758f","displayName":"Map a DataFrame to a pre-defined Schema","description":"This step maps a new DataFrame to a pre-defined spark schema","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"inputDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame that needs to be modified"},{"type":"object","name":"destinationSchema","required":true,"className":"com.acxiom.pipeline.steps.Schema","parameterType":"com.acxiom.pipeline.steps.Schema","description":"The schema that the new data should map to"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations","description":"The object with transform, alias, and filter logic details"},{"type":"boolean","name":"addNewColumns","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.mapDataFrameToSchema","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3ee74590-9131-43e1-8ee8-ad320482a592","displayName":"Merge a DataFrame to an existing DataFrame","description":"This step merges two DataFrames to create a single DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"inputDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The first DataFrame"},{"type":"text","name":"destinationDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The second DataFrame used as the driver"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations","description":"The object with transform, alias, and filter logic details"},{"type":"boolean","name":"addNewColumns","required":false,"parameterType":"Boolean"},{"type":"boolean","name":"distinct","required":false,"defaultValue":"true","parameterType":"Boolean","description":"Flag to determine whether a distinct union should be performed"}],"engineMeta":{"spark":"TransformationSteps.mergeDataFrames","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"ac3dafe4-e6ee-45c9-8fc6-fa7f918cf4f2","displayName":"Modify or Create Columns using Transforms Provided","description":"This step transforms existing columns and/or adds new columns to an existing dataframe using expressions provided","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The input DataFrame"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations"}],"engineMeta":{"spark":"TransformationSteps.applyTransforms","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3e2da5a8-387d-49b1-be22-c03764fb0fde","displayName":"Select Expressions","description":"Select each provided expresion from a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to select from"},{"type":"text","name":"expressions","required":true,"parameterType":"List[String]","description":"List of expressions to select"}],"engineMeta":{"spark":"TransformationSteps.selectExpressions","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"1e0a234a-8ae5-4627-be6d-3052b33d9014","displayName":"Add Column","description":"Add a new column to a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to add to"},{"type":"text","name":"columnName","required":true,"parameterType":"String","description":"The name of the new column"},{"type":"text","name":"expression","required":true,"parameterType":"String","description":"The expression used for the column"},{"type":"boolean","name":"standardizeColumnName","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.addColumn","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"08c9c5a9-a10d-477e-a702-19bd24889d1e","displayName":"Add Columns","description":"Add multiple new columns to a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to add to"},{"type":"text","name":"columns","required":true,"parameterType":"Map[String,String]","description":"A map of column names and expressions"},{"type":"boolean","name":"standardizeColumnNames","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.addColumns","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"42c328ac-a6bd-49ca-b597-b706956d294c","displayName":"Flatten a DataFrame","description":"This step will flatten all nested fields contained in a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to flatten"},{"type":"text","name":"separator","required":false,"defaultValue":"_","parameterType":"String","description":"Separator to place between nested field names"},{"type":"text","name":"fieldList","required":false,"parameterType":"List[String]","description":"List of fields to flatten. Will flatten all fields if left empty"},{"type":"integer","name":"depth","required":false,"parameterType":"Int","description":"How deep should we traverse when flattening."}],"engineMeta":{"spark":"TransformationSteps.flattenDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[_]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"a981080d-714c-4d36-8b09-d95842ec5655","displayName":"Standardize Column Names on a DataFrame","description":"This step will standardize columns names on existing DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.DataFrame"}],"engineMeta":{"spark":"TransformationSteps.standardizeColumnNames","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"541c4f7d-3524-4d53-bbd9-9f2cfd9d1bd1","displayName":"Save a Dataframe to a TempView","description":"This step stores an existing dataframe to a TempView to be used in future queries in the session","type":"Pipeline","category":"Query","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The dataframe to store"},{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to create (optional, random name will be created if not provided)"}],"engineMeta":{"spark":"QuerySteps.dataFrameToTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"71b71ef3-eaa7-4a1f-b3f3-603a1a54846d","displayName":"Create a TempView from a Query","description":"This step runs a SQL statement against existing TempViews from this session and returns a new TempView","type":"Pipeline","category":"Query","params":[{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"},{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to create (optional, random name will be created if not provided)"}],"engineMeta":{"spark":"QuerySteps.queryToTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"61378ed6-8a4f-4e6d-9c92-6863c9503a54","displayName":"Create a DataFrame from a Query","description":"This step runs a SQL statement against existing TempViews from this session and returns a new DataFrame","type":"Pipeline","category":"Query","params":[{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"}],"engineMeta":{"spark":"QuerySteps.queryToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"57b0e491-e09b-4428-aab2-cebe1f217eda","displayName":"Create a DataFrame from an Existing TempView","description":"This step pulls an existing TempView from this session into a new DataFrame","type":"Pipeline","category":"Query","params":[{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to use"}],"engineMeta":{"spark":"QuerySteps.tempViewToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"648f27aa-6e3b-44ed-a093-bc284783731b","displayName":"Create a TempView from a DataFrame Query","description":"This step runs a SQL statement against an existing DataFrame from this session and returns a new TempView","type":"Pipeline","category":"Query","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The dataframe to query"},{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"},{"type":"text","name":"inputViewName","required":true,"parameterType":"String","description":"The name to use when creating the view representing the input dataframe (same name used in query)"},{"type":"text","name":"outputViewName","required":false,"parameterType":"String","description":"The name of the view to create (optional, random name will be created if not provided)"}],"engineMeta":{"spark":"QuerySteps.dataFrameQueryToTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"dfb8a387-6245-4b1c-ae6c-94067eb83962","displayName":"Create a DataFrame from a DataFrame Query","description":"This step runs a SQL statement against an existing DataFrame from this session and returns a new DataFrame","type":"Pipeline","category":"Query","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The dataframe to query"},{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"},{"type":"text","name":"inputViewName","required":true,"parameterType":"String","description":"The name to use when creating the view representing the input dataframe (same name used in query)"}],"engineMeta":{"spark":"QuerySteps.dataFrameQueryToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"c88de095-14e0-4c67-8537-0325127e2bd2","displayName":"Cache an exising TempView","description":"This step will cache an existing TempView","type":"Pipeline","category":"Query","params":[{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to cache"}],"engineMeta":{"spark":"QuerySteps.cacheTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"0342654c-2722-56fe-ba22-e342169545af","displayName":"Copy (auto buffering)","description":"Copy the contents of the source path to the destination path. This function will call connect on both FileManagers.","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"c40169a3-1e77-51ab-9e0a-3f24fb98beef","displayName":"Copy (basic buffering)","description":"Copy the contents of the source path to the destination path using buffer sizes. This function will call connect on both FileManagers.","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to"},{"type":"text","name":"inputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for reading data during copy"},{"type":"text","name":"outputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for writing data during copy"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f5a24db0-e91b-5c88-8e67-ab5cff09c883","displayName":"Copy (advanced buffering)","description":"Copy the contents of the source path to the destination path using full buffer sizes. This function will call connect on both FileManagers.","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to"},{"type":"text","name":"inputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for reading data during copy"},{"type":"text","name":"outputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for writing data during copy"},{"type":"text","name":"copyBufferSize","required":true,"parameterType":"Int","description":"The intermediate buffer size to use during copy"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"1af68ab5-a3fe-4afb-b5fa-34e52f7c77f5","displayName":"Compare File Sizes","description":"Compare the file sizes of the source and destination paths","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to the source"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to th destination"}],"engineMeta":{"spark":"FileManagerSteps.compareFileSizes","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Int"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"bf2c4df8-a215-480b-87d8-586984e04189","displayName":"Delete (file)","description":"Delete a file","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The FileManager"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the file being deleted"}],"engineMeta":{"spark":"FileManagerSteps.deleteFile","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"3d1e8519-690c-55f0-bd05-1e7b97fb6633","displayName":"Disconnect a FileManager","description":"Disconnects a FileManager from the underlying file system","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The file manager to disconnect"}],"engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"259a880a-3e12-4843-9f02-2cfc2a05f576","displayName":"Create a FileManager","description":"Creates a FileManager using the provided FileConnector","type":"Pipeline","category":"Connectors","params":[{"type":"text","name":"fileConnector","required":true,"parameterType":"com.acxiom.pipeline.connectors.FileConnector","description":"The FileConnector to use to create the FileManager implementation"}],"engineMeta":{"spark":"FileManagerSteps.getFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.FileManager"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"9d467cb0-8b3d-40a0-9ccd-9cf8c5b6cb38","displayName":"Create SFTP FileManager","description":"Simple function to generate the SFTPFileManager for the remote SFTP file system","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"hostName","required":true,"parameterType":"String","description":"The name of the host to connect"},{"type":"text","name":"username","required":false,"parameterType":"String","description":"The username used for connection"},{"type":"text","name":"password","required":false,"parameterType":"String","description":"The password used for connection"},{"type":"integer","name":"port","required":false,"parameterType":"Int","description":"The optional port if other than 22"},{"type":"boolean","name":"strictHostChecking","required":false,"parameterType":"Boolean","description":"Option to automatically add keys to the known_hosts file. Default is false."}],"engineMeta":{"spark":"SFTPSteps.createFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.SFTPFileManager"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"22fcc0e7-0190-461c-a999-9116b77d5919","displayName":"Build a DataFrameReader Object","description":"This step will build a DataFrameReader object that can be used to read a file into a dataframe","type":"Pipeline","category":"InputOutput","params":[{"type":"object","name":"dataFrameReaderOptions","required":true,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"DataFrameSteps.getDataFrameReader","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrameReader"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"66a451c8-ffbd-4481-9c37-71777c3a240f","displayName":"Load Using DataFrameReader","description":"This step will load a DataFrame given a dataFrameReader.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrameReader","required":true,"parameterType":"org.apache.spark.sql.DataFrameReader","description":"The DataFrameReader to use when creating the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.load","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"d7cf27e6-9ca5-4a73-a1b3-d007499f235f","displayName":"Load DataFrame","description":"This step will load a DataFrame given a DataFrameReaderOptions object.","type":"Pipeline","category":"InputOutput","params":[{"type":"object","name":"dataFrameReaderOptions","required":true,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The DataFrameReaderOptions to use when creating the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.loadDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"8a00dcf8-e6a9-4833-871e-c1f3397ab378","displayName":"Build a DataFrameWriter Object","description":"This step will build a DataFrameWriter object that can be used to write a file into a dataframe","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to use when creating the DataFrameWriter"},{"type":"object","name":"options","required":true,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The DataFrameWriterOptions to use when writing the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.getDataFrameWriter","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrameWriter[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"9aa6ae9f-cbeb-4b36-ba6a-02eee0a46558","displayName":"Save Using DataFrameWriter","description":"This step will save a DataFrame given a dataFrameWriter[Row].","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrameWriter","required":true,"parameterType":"org.apache.spark.sql.DataFrameWriter[_]","description":"The DataFrameWriter to use when saving"}],"engineMeta":{"spark":"DataFrameSteps.save","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"e5ac3671-ee10-4d4e-8206-fec7effdf7b9","displayName":"Save DataFrame","description":"This step will save a DataFrame given a DataFrameWriterOptions object.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to save"},{"type":"object","name":"dataFrameWriterOptions","required":true,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The DataFrameWriterOptions to use for saving"}],"engineMeta":{"spark":"DataFrameSteps.saveDataFrame","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"fa05a970-476d-4617-be4d-950cfa65f2f8","displayName":"Persist DataFrame","description":"Persist a DataFrame to provided storage level.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to persist"},{"type":"text","name":"storageLevel","required":false,"parameterType":"String","description":"The optional storage mechanism to use when persisting the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.persistDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"e6fe074e-a1fa-476f-9569-d37295062186","displayName":"Unpersist DataFrame","description":"Unpersist a DataFrame.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to unpersist"},{"type":"boolean","name":"blocking","required":false,"parameterType":"Boolean","description":"Optional flag to indicate whether to block while unpersisting"}],"engineMeta":{"spark":"DataFrameSteps.unpersistDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"71323226-bcfd-4fa1-bf9e-24e455e41144","displayName":"RepartitionDataFrame","description":"Repartition a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to repartition"},{"type":"text","name":"partitions","required":true,"parameterType":"Int","description":"The number of partitions to use"},{"type":"boolean","name":"rangePartition","required":false,"parameterType":"Boolean","description":"Flag indicating whether to repartition by range. This takes precedent over the shuffle flag"},{"type":"boolean","name":"shuffle","required":false,"parameterType":"Boolean","description":"Flag indicating whether to perform a normal partition"},{"type":"text","name":"partitionExpressions","required":false,"parameterType":"List[String]","description":"The partition expressions to use"}],"engineMeta":{"spark":"DataFrameSteps.repartitionDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"5e0358a0-d567-5508-af61-c35a69286e4e","displayName":"Javascript Step","description":"Executes a script and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"javascript","parameterType":"String","description":"Javascript to execute"}],"engineMeta":{"spark":"JavascriptSteps.processScript","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"570c9a80-8bd1-5f0c-9ae0-605921fe51e2","displayName":"Javascript Step with single object provided","description":"Executes a script with single object provided and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"javascript","parameterType":"String","description":"Javascript script to execute"},{"type":"text","name":"value","required":true,"parameterType":"Any","description":"Value to bind to the script"}],"engineMeta":{"spark":"JavascriptSteps.processScriptWithValue","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]},{"id":"f92d4816-3c62-4c29-b420-f00994bfcd86","displayName":"Javascript Step with map of objects provided","description":"Executes a script with map of objects provided and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"javascript","parameterType":"String"},{"type":"text","name":"values","required":true,"parameterType":"Map[String,Any]","description":"Map of name/value pairs to bind to the script"},{"type":"boolean","name":"unwrapOptions","required":false,"parameterType":"Boolean","description":"Flag to control option unwrapping behavior"}],"engineMeta":{"spark":"JavascriptSteps.processScriptWithValues","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3-SNAPSHOT.jar"]}],"pkgObjs":[{"id":"com.acxiom.pipeline.steps.JDBCDataFrameReaderOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"JDBC Data Frame Reader Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"url\":{\"type\":\"string\"},\"table\":{\"type\":\"string\"},\"predicates\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"readerOptions\":{\"$ref\":\"#/definitions/DataFrameReaderOptions\"}},\"definitions\":{\"DataFrameReaderOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}},\"Schema\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}}},\"Attribute\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"name\":{\"type\":\"string\"},\"dataType\":{\"$ref\":\"#/definitions/AttributeType\"}}},\"AttributeType\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"baseType\":{\"type\":\"string\"},\"valueType\":{\"$ref\":\"#/definitions/AttributeType\"},\"nameType\":{\"$ref\":\"#/definitions/AttributeType\"},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}}}}"},{"id":"com.acxiom.pipeline.steps.DataFrameWriterOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Data Frame Writer Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"saveMode\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"bucketingOptions\":{\"$ref\":\"#/definitions/BucketingOptions\"},\"partitionBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"sortBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}},\"definitions\":{\"BucketingOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"numBuckets\":{\"type\":\"integer\"},\"columns\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}},\"required\":[\"numBuckets\"]}}}","template":"{\"form\":[{\"type\":\"select\",\"key\":\"format\",\"templateOptions\":{\"label\":\"Format\",\"placeholder\":\"\",\"valueProp\":\"value\",\"options\":[{\"value\":\"csv\",\"name\":\"CSV\"},{\"value\":\"json\",\"name\":\"JSON\"},{\"value\":\"parquet\",\"name\":\"Parquet\"},{\"value\":\"orc\",\"name\":\"Orc\"},{\"value\":\"text\",\"name\":\"Text\"}],\"labelProp\":\"name\",\"focus\":false,\"_flatOptions\":true,\"disabled\":false}},{\"key\":\"options.encoding\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Encoding\",\"placeholder\":\"\",\"focus\":false},\"expressionProperties\":{\"templateOptions.disabled\":\"!model.format\"}},{\"key\":\"saveMode\",\"type\":\"select\",\"defaultValue\":false,\"templateOptions\":{\"label\":\"Save Mode\",\"placeholder\":\"\",\"valueProp\":\"value\",\"options\":[{\"value\":\"OVERWRITE\",\"name\":\"Overwrite\"},{\"value\":\"append\",\"name\":\"Append\"},{\"value\":\"ignore\",\"name\":\"Ignore\"},{\"value\":\"error\",\"name\":\"Error\"},{\"value\":\"errorifexists\",\"name\":\"Error If Exists\"}],\"labelProp\":\"name\",\"focus\":false,\"_flatOptions\":true,\"disabled\":false}},{\"key\":\"partitionBy\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Partition By Columns\",\"placeholder\":\"Add column names to use during partitioning\"}},{\"key\":\"sortBy\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Sort By Columns\",\"placeholder\":\"Add column names to use during sorting\"}},{\"key\":\"bucketingOptions\",\"wrappers\":[\"panel\"],\"templateOptions\":{\"label\":\"Bucketing Options\"},\"fieldGroup\":[{\"key\":\"numBuckets\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Number of Buckets\",\"type\":\"number\"}},{\"key\":\"columns\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Bucket Columns\",\"placeholder\":\"Add column names to use during bucketing\"}}]},{\"key\":\"options.sep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.lineSep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Line Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'text' || model.format === 'json' ? false : true\"},{\"key\":\"options.escapeQuotes\",\"hideExpression\":\"model.format === 'csv' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Escape Quotes?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.quote\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Quote\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.escape\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Escape Character\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"}]}"},{"id":"com.acxiom.pipeline.steps.JDBCDataFrameWriterOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"JDBC Data Frame Writer Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"url\":{\"type\":\"string\"},\"table\":{\"type\":\"string\"},\"writerOptions\":{\"$ref\":\"#/definitions/DataFrameWriterOptions\"}},\"definitions\":{\"DataFrameWriterOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"saveMode\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"bucketingOptions\":{\"$ref\":\"#/definitions/BucketingOptions\"},\"partitionBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"sortBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}}},\"BucketingOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"numBuckets\":{\"type\":\"integer\"},\"columns\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}},\"required\":[\"numBuckets\"]}}}"},{"id":"com.acxiom.pipeline.steps.Transformations","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Transformations\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"columnDetails\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/ColumnDetails\"}},\"filter\":{\"type\":\"string\"},\"standardizeColumnNames\":{}},\"definitions\":{\"ColumnDetails\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"outputField\":{\"type\":\"string\"},\"inputAliases\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"expression\":{\"type\":\"string\"}}}}}","template":"{\"form\":[{\"key\":\"filter\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Filter\"}},{\"key\":\"standardizeColumnNames\",\"type\":\"checkbox\",\"defaultValue\":false,\"templateOptions\":{\"floatLabel\":\"always\",\"align\":\"start\",\"label\":\"Standardize Column Names?\",\"hideFieldUnderline\":true,\"color\":\"accent\",\"placeholder\":\"\",\"focus\":false,\"hideLabel\":true,\"disabled\":false,\"indeterminate\":true}},{\"fieldArray\":{\"fieldGroup\":[{\"key\":\"outputField\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Output Field\"}},{\"key\":\"inputAliases\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Input Alias\"}},{\"key\":\"expression\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Expression (Optional)\"}}]},\"key\":\"columnDetails\",\"wrappers\":[\"panel\"],\"type\":\"repeat\",\"templateOptions\":{\"label\":\"Column Details\"}}]}"},{"id":"com.acxiom.pipeline.steps.DataFrameReaderOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Data Frame Reader Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"schema\":{\"$ref\":\"#/definitions/Schema\"}},\"definitions\":{\"Schema\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}}},\"Attribute\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"name\":{\"type\":\"string\"},\"dataType\":{\"$ref\":\"#/definitions/AttributeType\"}}},\"AttributeType\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"baseType\":{\"type\":\"string\"},\"valueType\":{\"$ref\":\"#/definitions/AttributeType\"},\"nameType\":{\"$ref\":\"#/definitions/AttributeType\"},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}}}}","template":"{\"form\":[{\"type\":\"select\",\"key\":\"format\",\"templateOptions\":{\"label\":\"Format\",\"placeholder\":\"\",\"valueProp\":\"value\",\"options\":[{\"value\":\"csv\",\"name\":\"CSV\"},{\"value\":\"json\",\"name\":\"JSON\"},{\"value\":\"parquet\",\"name\":\"Parquet\"},{\"value\":\"orc\",\"name\":\"Orc\"},{\"value\":\"text\",\"name\":\"Text\"}],\"labelProp\":\"name\",\"focus\":false,\"_flatOptions\":true,\"disabled\":false}},{\"key\":\"options.encoding\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Encoding\",\"placeholder\":\"\",\"focus\":false},\"expressionProperties\":{\"templateOptions.disabled\":\"!model.format\"}},{\"key\":\"options.multiLine\",\"hideExpression\":\"model.format === 'csv' || model.format === 'json' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Multiline?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.header\",\"hideExpression\":\"model.format === 'csv' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Skip Header?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.sep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.lineSep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Line Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'text' || model.format === 'json' ? false : true\"},{\"key\":\"options.quote\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Quote\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.escape\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Escape Character\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.primitivesAsString\",\"hideExpression\":\"model.format === 'json' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Primitive As String?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.inferSchema\",\"hideExpression\":\"model.format === 'csv' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Infer Schema?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"expressionProperties\":{\"templateOptions.disabled\":\"model.options.inferSchema || model.format === 'json' ? false : true\"},\"key\":\"options.samplingRatio\",\"hideExpression\":\"model.format === 'csv' || model.format === 'json' ? false : true\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Sampling Ration\",\"type\":\"number\"}}]}"},{"id":"com.acxiom.pipeline.steps.Schema","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Schema\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}},\"definitions\":{\"Attribute\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"name\":{\"type\":\"string\"},\"dataType\":{\"$ref\":\"#/definitions/AttributeType\"}}},\"AttributeType\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"baseType\":{\"type\":\"string\"},\"valueType\":{\"$ref\":\"#/definitions/AttributeType\"},\"nameType\":{\"$ref\":\"#/definitions/AttributeType\"},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}},\"Schema\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}}}}}"}]} +{"pkgs":["com.acxiom.pipeline.steps"],"steps":[{"id":"3806f23b-478c-4054-b6c1-37f11db58d38","displayName":"Read a DataFrame from Table","description":"This step will read a dataFrame in a given format from the meta store","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"table","required":true,"parameterType":"String","description":"The name of the table to read"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The DataFrameReaderOptions to use"}],"engineMeta":{"spark":"CatalogSteps.readDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"e2b4c011-e71b-46f9-a8be-cf937abc2ec4","displayName":"Write DataFrame to Table","description":"This step will write a dataFrame in a given format to the meta store","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to write"},{"type":"text","name":"table","required":true,"parameterType":"String","description":"The name of the table to write to"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The DataFrameWriterOptions to use"}],"engineMeta":{"spark":"CatalogSteps.writeDataFrame","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"5874ab64-13c7-404c-8a4f-67ff3b0bc7cf","displayName":"Drop Catalog Object","description":"This step will drop an object from the meta store","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the object to drop"},{"type":"text","name":"objectType","required":false,"defaultValue":"TABLE","parameterType":"String","description":"Type of object to drop"},{"type":"boolean","name":"ifExists","required":false,"defaultValue":"false","parameterType":"Boolean","description":"Flag to control whether existence is checked"},{"type":"boolean","name":"cascade","required":false,"defaultValue":"false","parameterType":"Boolean","description":"Flag to control whether this deletion should cascade"}],"engineMeta":{"spark":"CatalogSteps.drop","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"17be71f9-1492-4404-a355-1cc973694cad","displayName":"Database Exists","description":"Check spark catalog for a database with the given name.","type":"branch","category":"Decision","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the database"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"CatalogSteps.databaseExists","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"95181811-d83e-4136-bedb-2cba1de90301","displayName":"Table Exists","description":"Check spark catalog for a table with the given name.","type":"branch","category":"Decision","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the table"},{"type":"text","name":"database","required":false,"parameterType":"String","description":"Name of the database"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"CatalogSteps.tableExists","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"f4adfe70-2ae3-4b8d-85d1-f53e91c8dfad","displayName":"Set Current Database","description":"Set the current default database for the spark session.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the database"}],"engineMeta":{"spark":"CatalogSteps.setCurrentDatabase","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"663f8c93-0a42-4c43-8263-33f89c498760","displayName":"Create Table","description":"Create a table in the meta store.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"name","required":true,"parameterType":"String","description":"Name of the table"},{"type":"text","name":"externalPath","required":false,"parameterType":"String","description":"Path of the external table"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"Options containing the format, schema, and settings"}],"engineMeta":{"spark":"CatalogSteps.createTable","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"836aab38-1140-4606-ab73-5b6744f0e7e7","displayName":"Load","description":"This step will create a DataFrame using the given DataConnector","type":"Pipeline","category":"Connectors","params":[{"type":"text","name":"connector","required":true,"parameterType":"com.acxiom.pipeline.connectors.DataConnector","description":"The data connector to use when writing"},{"type":"text","name":"source","required":false,"parameterType":"String","description":"The source path to load data"},{"type":"object","name":"readOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The optional options to use while reading the data"}],"engineMeta":{"spark":"DataConnectorSteps.loadDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"5608eba7-e9ff-48e6-af77-b5e810b99d89","displayName":"Write","description":"This step will write a DataFrame using the given DataConnector","type":"Pipeline","category":"Connectors","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.DataFrame","description":"The DataFrame to write"},{"type":"text","name":"connector","required":true,"parameterType":"com.acxiom.pipeline.connectors.DataConnector","description":"The data connector to use when writing"},{"type":"text","name":"destination","required":false,"parameterType":"String","description":"The destination path to write data"},{"type":"object","name":"writeOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The optional DataFrame options to use while writing"}],"engineMeta":{"spark":"DataConnectorSteps.writeDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.streaming.StreamingQuery"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"87db259d-606e-46eb-b723-82923349640f","displayName":"Load DataFrame from HDFS path","description":"This step will read a dataFrame from the given HDFS path","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"path","required":true,"parameterType":"String","description":"The HDFS path to load data into the DataFrame"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"HDFSSteps.readFromPath","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"8daea683-ecde-44ce-988e-41630d251cb8","displayName":"Load DataFrame from HDFS paths","description":"This step will read a dataFrame from the given HDFS paths","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"paths","required":true,"parameterType":"List[String]","description":"The HDFS paths to load data into the DataFrame"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"HDFSSteps.readFromPaths","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"0a296858-e8b7-43dd-9f55-88d00a7cd8fa","displayName":"Write DataFrame to HDFS","description":"This step will write a dataFrame in a given format to HDFS","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to write"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The GCS path to write data"},{"type":"object","name":"options","required":false,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The optional DataFrame Options"}],"engineMeta":{"spark":"HDFSSteps.writeToPath","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"e4dad367-a506-5afd-86c0-82c2cf5cd15c","displayName":"Create HDFS FileManager","description":"Simple function to generate the HDFSFileManager for the local HDFS file system","type":"Pipeline","category":"InputOutput","params":[],"engineMeta":{"spark":"HDFSSteps.createFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.HDFSFileManager"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"a7e17c9d-6956-4be0-a602-5b5db4d1c08b","displayName":"Scala script Step","description":"Executes a script and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"scala","parameterType":"String","description":"A scala script to execute"}],"engineMeta":{"spark":"ScalaSteps.processScript","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"8bf8cef6-cf32-4d85-99f4-e4687a142f84","displayName":"Scala script Step with additional object provided","description":"Executes a script with the provided object and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"scala","parameterType":"String","description":"A scala script to execute"},{"type":"text","name":"value","required":true,"parameterType":"Any","description":"A value to pass to the script"},{"type":"text","name":"type","required":false,"parameterType":"String","description":"The type of the value to pass to the script"}],"engineMeta":{"spark":"ScalaSteps.processScriptWithValue","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"3ab721e8-0075-4418-aef1-26abdf3041be","displayName":"Scala script Step with additional objects provided","description":"Executes a script with the provided object and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"scala","parameterType":"String","description":"A scala script to execute"},{"type":"object","name":"values","required":true,"parameterType":"Map[String,Any]","description":"Map of name/value pairs that will be bound to the script"},{"type":"object","name":"types","required":false,"parameterType":"Map[String,String]","description":"Map of type overrides for the values provided"},{"type":"boolean","name":"unwrapOptions","required":false,"parameterType":"Boolean","description":"Flag to toggle option unwrapping behavior"}],"engineMeta":{"spark":"ScalaSteps.processScriptWithValues","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"6e42b0c3-340e-4848-864c-e1b5c57faa4f","displayName":"Join DataFrames","description":"Join two dataFrames together.","type":"Pipeline","category":"Data","params":[{"type":"text","name":"left","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"Left side of the join"},{"type":"text","name":"right","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"Right side of the join"},{"type":"text","name":"expression","required":false,"parameterType":"String","description":"Join expression. Optional for cross joins"},{"type":"text","name":"leftAlias","required":false,"defaultValue":"left","parameterType":"String","description":"Left side alias"},{"type":"text","name":"rightAlias","required":false,"defaultValue":"right","parameterType":"String","description":"Right side alias"},{"type":"text","name":"joinType","required":false,"defaultValue":"inner","parameterType":"String","description":"Type of join to perform"}],"engineMeta":{"spark":"DataSteps.join","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"823eeb28-ec81-4da6-83f2-24a1e580b0e5","displayName":"Group By","description":"Group by a list of grouping expressions and a list of aggregates.","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to group"},{"type":"text","name":"groupings","required":true,"parameterType":"List[String]","description":"List of expressions to group by"},{"type":"text","name":"aggregations","required":true,"parameterType":"List[String]","description":"List of aggregations to apply"}],"engineMeta":{"spark":"DataSteps.groupBy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"d322769c-18a0-49c2-9875-41446892e733","displayName":"Union","description":"Union two DataFrames together.","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The initial DataFrame"},{"type":"text","name":"append","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The dataFrame to append"},{"type":"boolean","name":"distinct","required":false,"defaultValue":"true","parameterType":"Boolean","description":"Flag to control distinct behavior"}],"engineMeta":{"spark":"DataSteps.union","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"80583aa9-41b7-4906-8357-cc2d3670d970","displayName":"Add a Column with a Static Value to All Rows in a DataFrame (metalus-common)","description":"This step will add a column with a static value to all rows in the provided data frame","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The data frame to add the column"},{"type":"text","name":"columnName","required":true,"parameterType":"String","description":"The name to provide the id column"},{"type":"text","name":"columnValue","required":true,"parameterType":"Any","description":"The name of the new column"},{"type":"boolean","name":"standardizeColumnName","required":false,"defaultValue":"true","parameterType":"Boolean","description":"The value to add"}],"engineMeta":{"spark":"DataSteps.addStaticColumnToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"e625eed6-51f0-44e7-870b-91c960cdc93d","displayName":"Adds a Unique Identifier to a DataFrame (metalus-common)","description":"This step will add a new unique identifier to an existing data frame using the monotonically_increasing_id method","type":"Pipeline","category":"Data","params":[{"type":"text","name":"idColumnName","required":true,"parameterType":"String","description":"The name to provide the id column"},{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The data frame to add the column"}],"engineMeta":{"spark":"DataSteps.addUniqueIdToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"fa0fcabb-d000-4a5e-9144-692bca618ddb","displayName":"Filter a DataFrame","description":"This step will filter a DataFrame based on the where expression provided","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to filter"},{"type":"text","name":"expression","required":true,"parameterType":"String","description":"The expression to apply to the DataFrame to filter rows"}],"engineMeta":{"spark":"DataSteps.applyFilter","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"5d0d7c5c-c287-4565-80b2-2b1a847b18c6","displayName":"Get DataFrame Count","description":"Get a count of records in a DataFrame.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to count"}],"engineMeta":{"spark":"DataSteps.getDataFrameCount","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Long"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"252b6086-da45-4042-a9a8-31ebf57948af","displayName":"Drop Duplicate Records","description":"Drop duplicate records from a DataFrame","type":"Pipeline","category":"Data","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to drop duplicate records from"},{"type":"text","name":"columnNames","required":true,"parameterType":"List[String]","description":"Columns to use for determining distinct values to drop"}],"engineMeta":{"spark":"DataSteps.dropDuplicateRecords","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"d5ac88a2-caa2-473c-a9f7-ffb0269880b2","displayName":"Rename Column","description":"Rename a column on a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to change"},{"type":"text","name":"oldColumnName","required":true,"parameterType":"String","description":"The name of the column you want to change"},{"type":"text","name":"newColumnName","required":true,"parameterType":"String","description":"The new name to give the column"}],"engineMeta":{"spark":"DataSteps.renameColumn","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"6ed36f89-35d1-4280-a555-fbcd8dd76bf2","displayName":"Retry (simple)","description":"Makes a decision to retry or stop based on a named counter","type":"branch","category":"RetryLogic","params":[{"type":"text","name":"counterName","required":true,"parameterType":"String","description":"The name of the counter to use for tracking"},{"type":"text","name":"maxRetries","required":true,"parameterType":"Int","description":"The maximum number of retries allowed"},{"type":"result","name":"retry","required":false},{"type":"result","name":"stop","required":false}],"engineMeta":{"spark":"FlowUtilsSteps.simpleRetry","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"a2f3e151-cb81-4c69-8475-c1a287bbb4cb","displayName":"Convert CSV String Dataset to DataFrame","description":"This step will convert the provided CSV string Dataset into a DataFrame that can be passed to other steps","type":"Pipeline","category":"CSV","params":[{"type":"text","name":"dataset","required":true,"parameterType":"org.apache.spark.sql.Dataset[String]","description":"The dataset containing CSV strings"},{"type":"object","name":"dataFrameReaderOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The CSV parsing options"}],"engineMeta":{"spark":"CSVSteps.csvDatasetToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"d25209c1-53f6-49ad-a402-257ae756ac2a","displayName":"Convert CSV String to DataFrame","description":"This step will convert the provided CSV string into a DataFrame that can be passed to other steps","type":"Pipeline","category":"CSV","params":[{"type":"text","name":"csvString","required":true,"parameterType":"String","description":"The csv string to convert to a DataFrame"},{"type":"text","name":"delimiter","required":false,"defaultValue":",","parameterType":"String","description":"The field delimiter"},{"type":"text","name":"recordDelimiter","required":false,"defaultValue":"\\n","parameterType":"String","description":"The record delimiter"},{"type":"boolean","name":"header","required":false,"defaultValue":"false","parameterType":"Boolean","description":"Build header from the first row"}],"engineMeta":{"spark":"CSVSteps.csvStringToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"15889487-fd1c-4c44-b8eb-973c12f91fae","displayName":"Creates an HttpRestClient","description":"This step will build an HttpRestClient using a host url and optional authorization object","type":"Pipeline","category":"API","params":[{"type":"text","name":"hostUrl","required":true,"parameterType":"String","description":"The URL to connect including port"},{"type":"text","name":"authorization","required":false,"parameterType":"com.acxiom.pipeline.api.Authorization","description":"The optional authorization class to use when making connections"},{"type":"boolean","name":"allowSelfSignedCertificates","required":false,"parameterType":"Boolean","description":"Flag to allow using self signed certificates for http calls"}],"engineMeta":{"spark":"ApiSteps.createHttpRestClient","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.api.HttpRestClient"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"fcfd4b91-9a9c-438c-8afa-9f14c1e52a82","displayName":"Creates an HttpRestClient from protocol, host and port","description":"This step will build an HttpRestClient using url parts and optional authorization object","type":"Pipeline","category":"API","params":[{"type":"text","name":"protocol","required":true,"parameterType":"String","description":"The protocol to use when constructing the URL"},{"type":"text","name":"host","required":true,"parameterType":"String","description":"The host name to use when constructing the URL"},{"type":"text","name":"port","required":true,"parameterType":"Int","description":"The port to use when constructing the URL"},{"type":"text","name":"authorization","required":false,"parameterType":"com.acxiom.pipeline.api.Authorization","description":"The optional authorization class to use when making connections"}],"engineMeta":{"spark":"ApiSteps.createHttpRestClientFromParameters","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.api.HttpRestClient"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"b59f0486-78aa-4bd4-baf5-5c7d7c648ff0","displayName":"Check Path Exists","description":"Checks the path to determine whether it exists or not.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to verify"}],"engineMeta":{"spark":"ApiSteps.exists","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"7521ac47-84ec-4e50-b087-b9de4bf6d514","displayName":"Get the last modified date","description":"Gets the last modified date for the provided path","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource to get the last modified date"}],"engineMeta":{"spark":"ApiSteps.getLastModifiedDate","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.util.Date"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"fff7f7b6-5d9a-40b3-8add-6432552920a8","displayName":"Get Path Content Length","description":"Get the size of the content at the given path.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource to get the content length"}],"engineMeta":{"spark":"ApiSteps.getContentLength","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Long"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"dd351d47-125d-47fa-bafd-203bebad82eb","displayName":"Get Path Headers","description":"Get the headers for the content at the given path.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to get the headers"}],"engineMeta":{"spark":"ApiSteps.getHeaders","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Map[String,List[String]]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"532f72dd-8443-481d-8406-b74cdc08e342","displayName":"Delete Content","description":"Attempts to delete the provided path..","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to delete"}],"engineMeta":{"spark":"ApiSteps.delete","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"3b91e6e8-ec18-4468-9089-8474f4b4ba48","displayName":"GET String Content","description":"Retrieves the value at the provided path as a string.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to resource"}],"engineMeta":{"spark":"ApiSteps.getStringContent","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"34c2fc9a-2502-4c79-a0cb-3f866a0a0d6e","displayName":"POST String Content","description":"POSTs the provided string to the provided path using the content type and returns the response as a string.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to post the content"},{"type":"text","name":"content","required":true,"parameterType":"String","description":"The content to post"},{"type":"text","name":"contentType","required":false,"parameterType":"String","description":"The content type being sent to the path"}],"engineMeta":{"spark":"ApiSteps.postStringContent","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"49ae38b3-cb41-4153-9111-aa6aacf6721d","displayName":"PUT String Content","description":"PUTs the provided string to the provided path using the content type and returns the response as a string.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to post the content"},{"type":"text","name":"content","required":true,"parameterType":"String","description":"The content to put"},{"type":"text","name":"contentType","required":false,"parameterType":"String","description":"The content type being sent to the path"}],"engineMeta":{"spark":"ApiSteps.putStringContent","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"99b20c23-722f-4862-9f47-bc9f72440ae6","displayName":"GET Input Stream","description":"Creates a buffered input stream for the provided path","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource"},{"type":"text","name":"bufferSize","required":false,"parameterType":"Int","description":"The size of buffer to use with the stream"}],"engineMeta":{"spark":"ApiSteps.getInputStream","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.io.InputStream"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"f4120b1c-91df-452f-9589-b77f8555ba44","displayName":"GET Output Stream","description":"Creates a buffered output stream for the provided path.","type":"Pipeline","category":"API","params":[{"type":"text","name":"httpRestClient","required":true,"parameterType":"com.acxiom.pipeline.api.HttpRestClient","description":"The HttpRestClient to use when accessing the provided path"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the resource"},{"type":"text","name":"bufferSize","required":false,"parameterType":"Int","description":"The size of buffer to use with the stream"}],"engineMeta":{"spark":"ApiSteps.getOutputStream","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.io.OutputStream"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"cdb332e3-9ea4-4c96-8b29-c1d74287656c","displayName":"Load table as DataFrame using JDBCOptions","description":"This step will load a table from the provided JDBCOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"jdbcOptions","required":true,"parameterType":"org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions","description":"The options to use when loading the DataFrame"}],"engineMeta":{"spark":"JDBCSteps.readWithJDBCOptions","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"72dbbfc8-bd1d-4ce4-ab35-28fa8385ea54","displayName":"Load table as DataFrame using StepOptions","description":"This step will load a table from the provided JDBCDataFrameReaderOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"object","name":"jDBCStepsOptions","required":true,"className":"com.acxiom.pipeline.steps.JDBCDataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.JDBCDataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"JDBCSteps.readWithStepOptions","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"dcc57409-eb91-48c0-975b-ca109ba30195","displayName":"Load table as DataFrame","description":"This step will load a table from the provided jdbc information","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"url","required":true,"parameterType":"String","description":"A valid jdbc url"},{"type":"text","name":"table","required":true,"parameterType":"String","description":"A table name or subquery"},{"type":"text","name":"predicates","required":false,"parameterType":"List[String]","description":"Optional predicates used for partitioning"},{"type":"text","name":"connectionProperties","required":false,"parameterType":"Map[String,String]","description":"Optional properties for the jdbc connection"}],"engineMeta":{"spark":"JDBCSteps.readWithProperties","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"c9fddf52-34b1-4216-a049-10c33ccd24ab","displayName":"Write DataFrame to table using JDBCOptions","description":"This step will write a DataFrame as a table using JDBCOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to be written"},{"type":"text","name":"jdbcOptions","required":true,"parameterType":"org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions","description":"Options for configuring the JDBC connection"},{"type":"text","name":"saveMode","required":false,"parameterType":"String","description":"The value for the mode option. Defaulted to Overwrite"}],"engineMeta":{"spark":"JDBCSteps.writeWithJDBCOptions","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"77ffcd02-fbd0-4f79-9b35-ac9dc5fb7190","displayName":"Write DataFrame to table","description":"This step will write a DataFrame to a table using the provided properties","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to be written"},{"type":"text","name":"url","required":true,"parameterType":"String","description":"A valid jdbc url"},{"type":"text","name":"table","required":true,"parameterType":"String","description":"A table name or subquery"},{"type":"text","name":"connectionProperties","required":false,"parameterType":"Map[String,String]","description":"Optional properties for the jdbc connection"},{"type":"text","name":"saveMode","required":false,"parameterType":"String","description":"The value for the mode option. Defaulted to Overwrite"}],"engineMeta":{"spark":"JDBCSteps.writeWithProperties","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"3d6b77a1-52c2-49ba-99a0-7ec773dac696","displayName":"Write DataFrame to JDBC table","description":"This step will write a DataFrame to a table using the provided JDBCDataFrameWriterOptions","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to be written"},{"type":"object","name":"jDBCStepsOptions","required":true,"className":"com.acxiom.pipeline.steps.JDBCDataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.JDBCDataFrameWriterOptions","description":"Options for the JDBC connect and spark DataFrameWriter"}],"engineMeta":{"spark":"JDBCSteps.writeWithStepOptions","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"713fff3d-d407-4970-89ae-7844e6fc60e3","displayName":"Get JDBC Connection","description":"Get a jdbc connection.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"url","required":true,"parameterType":"String","description":"A valid jdbc url"},{"type":"text","name":"properties","required":false,"parameterType":"Map[String,String]","description":"Optional properties for the jdbc connection"}],"engineMeta":{"spark":"JDBCSteps.getConnection","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"java.sql.Connection"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"549828be-3d96-4561-bf94-7ad420f9d203","displayName":"Execute Sql","description":"Execute a sql command using jdbc.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"sql","required":true,"parameterType":"String","description":"Sql command to execute"},{"type":"text","name":"connection","required":true,"parameterType":"java.sql.Connection","description":"An open jdbc connection"},{"type":"text","name":"parameters","required":false,"parameterType":"List[Any]","description":"Optional list of bind variables"}],"engineMeta":{"spark":"JDBCSteps.executeSql","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"9c8957a3-899e-4f32-830e-d120b1917aa1","displayName":"Close JDBC Connection","description":"Close a JDBC Connection.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"connection","required":true,"parameterType":"java.sql.Connection","description":"An open jdbc connection"}],"engineMeta":{"spark":"JDBCSteps.closeConnection","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"3464dc85-5111-40fc-9bfb-1fd6fc8a2c17","displayName":"Convert JSON String to Map","description":"This step will convert the provided JSON string into a Map that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonString","required":true,"parameterType":"String","description":"The JSON string to convert to a map"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonStringToMap","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Map[String,Any]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"f4d19691-779b-4962-a52b-ee5d9a99068e","displayName":"Convert JSON Map to JSON String","description":"This step will convert the provided JSON map into a JSON string that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonMap","required":true,"parameterType":"Map[String,Any]","description":"The JSON map to convert to a JSON string"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonMapToString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"1f23eb37-98ee-43c2-ac78-17b04db3cc8d","displayName":"Convert object to JSON String","description":"This step will convert the provided object into a JSON string that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"obj","required":true,"parameterType":"AnyRef","description":"The object to convert to a JSON string"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.objectToJsonString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"880c5151-f7cd-40bb-99f2-06dbb20a6523","displayName":"Convert JSON String to object","description":"This step will convert the provided JSON string into an object that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonString","required":true,"parameterType":"String","description":"The JSON string to convert to an object"},{"type":"text","name":"objectName","required":true,"parameterType":"String","description":"The fully qualified class name of the object"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonStringToObject","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Any"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"68958a29-aab5-4f7e-9ffd-af99c33c512b","displayName":"Convert JSON String to Schema","description":"This step will convert the provided JSON string into a Schema that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"schema","required":true,"parameterType":"String","description":"The JSON string to convert to a Schema"},{"type":"text","name":"formats","required":false,"parameterType":"org.json4s.Formats","description":"Json4s Formats object that will override the pipeline context formats"}],"engineMeta":{"spark":"JSONSteps.jsonStringToSchema","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.Schema"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"cf4e9e6c-98d6-4a14-ae74-52322782c504","displayName":"Convert JSON String to DataFrame","description":"This step will convert the provided JSON string into a DataFrame that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"jsonString","required":true,"parameterType":"String","description":"The JSON string to convert to a DataFrame"}],"engineMeta":{"spark":"JSONSteps.jsonStringToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"d5cd835e-5e8f-49c0-9706-746d5a4d7b3a","displayName":"Convert JSON String Dataset to DataFrame","description":"This step will convert the provided JSON string Dataset into a DataFrame that can be passed to other steps","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"dataset","required":true,"parameterType":"org.apache.spark.sql.Dataset[String]","description":"The dataset containing JSON strings"},{"type":"object","name":"dataFrameReaderOptions","required":false,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The JSON parsing options"}],"engineMeta":{"spark":"JSONSteps.jsonDatasetToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"f3891201-5138-4cab-aebc-bcc319228543","displayName":"Build JSON4S Formats","description":"This step will build a json4s Formats object that can be used to override the default","type":"Pipeline","category":"JSON","params":[{"type":"text","name":"customSerializers","required":false,"parameterType":"List[com.acxiom.pipeline.applications.ClassInfo]","description":"List of custom serializer classes"},{"type":"text","name":"enumIdSerializers","required":false,"parameterType":"List[com.acxiom.pipeline.applications.ClassInfo]","description":"List of Enumeration classes to serialize by id"},{"type":"text","name":"enumNameSerializers","required":false,"parameterType":"List[com.acxiom.pipeline.applications.ClassInfo]","description":"List of Enumeration classes to serialize by name"}],"engineMeta":{"spark":"JSONSteps.buildJsonFormats","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.json4s.Formats"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"b5485d97-d4e8-41a6-8af7-9ce79a435140","displayName":"To String","description":"Returns the result of the toString method, can unwrap options","type":"Pipeline","category":"String","params":[{"type":"text","name":"value","required":true,"parameterType":"Any","description":"The value to convert"},{"type":"boolean","name":"unwrapOption","required":false,"parameterType":"Boolean","description":"Boolean indicating whether to unwrap the value from an Option prior to calling toString"}],"engineMeta":{"spark":"StringSteps.toString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"78e817ec-2bf2-4cbe-acba-e5bc9bdcffc5","displayName":"List To String","description":"Returns the result of the mkString method","type":"Pipeline","category":"String","params":[{"type":"text","name":"list","required":true,"parameterType":"List[Any]","description":"The list to convert"},{"type":"text","name":"separator","required":false,"parameterType":"String","description":"Separator character to use when making the string"},{"type":"boolean","name":"unwrapOptions","required":false,"parameterType":"Boolean","description":"Boolean indicating whether to unwrap each value from an Option"}],"engineMeta":{"spark":"StringSteps.listToString","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"fcd6b5fe-08ed-4cfd-acfe-eb676d7f4ecd","displayName":"To Lowercase","description":"Returns a lowercase string","type":"Pipeline","category":"String","params":[{"type":"text","name":"value","required":true,"parameterType":"String","description":"The value to lowercase"}],"engineMeta":{"spark":"StringSteps.toLowerCase","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"2f31ebf1-4ae2-4e04-9b29-4802cac8a198","displayName":"To Uppercase","description":"Returns an uppercase string","type":"Pipeline","category":"String","params":[{"type":"text","name":"value","required":true,"parameterType":"String","description":"The value to uppercase"}],"engineMeta":{"spark":"StringSteps.toUpperCase","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"96b7b521-5304-4e63-8435-63d84a358368","displayName":"String Split","description":"Returns a list of strings split off of the given string","type":"Pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to split"},{"type":"text","name":"regex","required":true,"parameterType":"String","description":"Regex to use when splitting the string"},{"type":"integer","name":"limit","required":false,"parameterType":"Int","description":"Max number elements to return in the list"}],"engineMeta":{"spark":"StringSteps.stringSplit","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"List[String]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"f75abedd-4aee-4979-8d56-ea7b0c1a86e1","displayName":"Substring","description":"Returns a substring","type":"Pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to parse"},{"type":"text","name":"begin","required":true,"parameterType":"Int","description":"The beginning index"},{"type":"integer","name":"end","required":false,"parameterType":"Int","description":"The end index"}],"engineMeta":{"spark":"StringSteps.substring","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"3fabf9ec-5383-4eb3-81af-6092ab7c370d","displayName":"String Equals","description":"Return whether string1 equals string2","type":"branch","category":"Decision","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to compare"},{"type":"text","name":"anotherString","required":true,"parameterType":"String","description":"The other string to compare"},{"type":"boolean","name":"caseInsensitive","required":false,"parameterType":"Boolean","description":"Boolean flag to indicate case sensitive compare"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"StringSteps.stringEquals","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"ff0562f5-2917-406d-aa78-c5d49ba6b99f","displayName":"String Matches","description":"Return whether string matches a given regex","type":"branch","category":"Decision","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to match"},{"type":"text","name":"regex","required":true,"parameterType":"String","description":"Regex to use for the match"},{"type":"result","name":"true","required":false},{"type":"result","name":"false","required":false}],"engineMeta":{"spark":"StringSteps.stringMatches","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"416baf4e-a1dd-49fc-83a9-0f41b77e57b7","displayName":"String Replace All","description":"Perform a literal or regex replacement on a string","type":"pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to modify"},{"type":"text","name":"matchString","required":true,"parameterType":"String","description":"The string to match"},{"type":"text","name":"replacement","required":false,"parameterType":"String","description":"The replacement string"},{"type":"boolean","name":"literal","required":false,"parameterType":"Boolean","description":"Perform \\'literal\\' match replacement"}],"engineMeta":{"spark":"StringSteps.stringReplaceAll","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"95438b82-8d50-41da-8094-c92449b9e7df","displayName":"String Replace First","description":"Perform a literal or regex replacement on the first occurrence in a string","type":"pipeline","category":"String","params":[{"type":"text","name":"string","required":true,"parameterType":"String","description":"The string to modify"},{"type":"text","name":"matchString","required":true,"parameterType":"String","description":"The string to match"},{"type":"text","name":"replacement","required":false,"parameterType":"String","description":"The replacement string"},{"type":"boolean","name":"literal","required":false,"parameterType":"Boolean","description":"Perform \\'literal\\' match replacement"}],"engineMeta":{"spark":"StringSteps.stringReplaceFirst","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"86c84fa3-ad45-4a49-ac05-92385b8e9572","displayName":"Get Credential","description":"This step provides access to credentials through the CredentialProvider","type":"Pipeline","category":"Credentials","params":[{"type":"text","name":"credentialName","required":true,"parameterType":"String","description":"The dataset containing CSV strings"}],"engineMeta":{"spark":"CredentialSteps.getCredential","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.Credential"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"219c787a-f502-4efc-b15d-5beeff661fc0","displayName":"Map a DataFrame to an existing DataFrame","description":"This step maps a new DataFrame to an existing DataFrame to make them compatible","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"inputDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame that needs to be modified"},{"type":"text","name":"destinationDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame that the new data needs to map to"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations","description":"The object with transform, alias, and filter logic details"},{"type":"boolean","name":"addNewColumns","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.mapToDestinationDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"8f9c08ea-4882-4265-bac7-2da3e942758f","displayName":"Map a DataFrame to a pre-defined Schema","description":"This step maps a new DataFrame to a pre-defined spark schema","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"inputDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame that needs to be modified"},{"type":"object","name":"destinationSchema","required":true,"className":"com.acxiom.pipeline.steps.Schema","parameterType":"com.acxiom.pipeline.steps.Schema","description":"The schema that the new data should map to"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations","description":"The object with transform, alias, and filter logic details"},{"type":"boolean","name":"addNewColumns","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.mapDataFrameToSchema","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"3ee74590-9131-43e1-8ee8-ad320482a592","displayName":"Merge a DataFrame to an existing DataFrame","description":"This step merges two DataFrames to create a single DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"inputDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The first DataFrame"},{"type":"text","name":"destinationDataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The second DataFrame used as the driver"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations","description":"The object with transform, alias, and filter logic details"},{"type":"boolean","name":"addNewColumns","required":false,"parameterType":"Boolean"},{"type":"boolean","name":"distinct","required":false,"defaultValue":"true","parameterType":"Boolean","description":"Flag to determine whether a distinct union should be performed"}],"engineMeta":{"spark":"TransformationSteps.mergeDataFrames","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"ac3dafe4-e6ee-45c9-8fc6-fa7f918cf4f2","displayName":"Modify or Create Columns using Transforms Provided","description":"This step transforms existing columns and/or adds new columns to an existing dataframe using expressions provided","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The input DataFrame"},{"type":"object","name":"transforms","required":true,"className":"com.acxiom.pipeline.steps.Transformations","parameterType":"com.acxiom.pipeline.steps.Transformations"}],"engineMeta":{"spark":"TransformationSteps.applyTransforms","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"3e2da5a8-387d-49b1-be22-c03764fb0fde","displayName":"Select Expressions","description":"Select each provided expresion from a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to select from"},{"type":"text","name":"expressions","required":true,"parameterType":"List[String]","description":"List of expressions to select"}],"engineMeta":{"spark":"TransformationSteps.selectExpressions","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"1e0a234a-8ae5-4627-be6d-3052b33d9014","displayName":"Add Column","description":"Add a new column to a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to add to"},{"type":"text","name":"columnName","required":true,"parameterType":"String","description":"The name of the new column"},{"type":"text","name":"expression","required":true,"parameterType":"String","description":"The expression used for the column"},{"type":"boolean","name":"standardizeColumnName","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.addColumn","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"08c9c5a9-a10d-477e-a702-19bd24889d1e","displayName":"Add Columns","description":"Add multiple new columns to a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to add to"},{"type":"text","name":"columns","required":true,"parameterType":"Map[String,String]","description":"A map of column names and expressions"},{"type":"boolean","name":"standardizeColumnNames","required":false,"parameterType":"Boolean"}],"engineMeta":{"spark":"TransformationSteps.addColumns","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"42c328ac-a6bd-49ca-b597-b706956d294c","displayName":"Flatten a DataFrame","description":"This step will flatten all nested fields contained in a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to flatten"},{"type":"text","name":"separator","required":false,"defaultValue":"_","parameterType":"String","description":"Separator to place between nested field names"},{"type":"text","name":"fieldList","required":false,"parameterType":"List[String]","description":"List of fields to flatten. Will flatten all fields if left empty"},{"type":"integer","name":"depth","required":false,"parameterType":"Int","description":"How deep should we traverse when flattening."}],"engineMeta":{"spark":"TransformationSteps.flattenDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[_]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"a981080d-714c-4d36-8b09-d95842ec5655","displayName":"Standardize Column Names on a DataFrame","description":"This step will standardize columns names on existing DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.DataFrame"}],"engineMeta":{"spark":"TransformationSteps.standardizeColumnNames","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"541c4f7d-3524-4d53-bbd9-9f2cfd9d1bd1","displayName":"Save a Dataframe to a TempView","description":"This step stores an existing dataframe to a TempView to be used in future queries in the session","type":"Pipeline","category":"Query","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The dataframe to store"},{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to create (optional, random name will be created if not provided)"}],"engineMeta":{"spark":"QuerySteps.dataFrameToTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"71b71ef3-eaa7-4a1f-b3f3-603a1a54846d","displayName":"Create a TempView from a Query","description":"This step runs a SQL statement against existing TempViews from this session and returns a new TempView","type":"Pipeline","category":"Query","params":[{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"},{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to create (optional, random name will be created if not provided)"}],"engineMeta":{"spark":"QuerySteps.queryToTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"61378ed6-8a4f-4e6d-9c92-6863c9503a54","displayName":"Create a DataFrame from a Query","description":"This step runs a SQL statement against existing TempViews from this session and returns a new DataFrame","type":"Pipeline","category":"Query","params":[{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"}],"engineMeta":{"spark":"QuerySteps.queryToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"57b0e491-e09b-4428-aab2-cebe1f217eda","displayName":"Create a DataFrame from an Existing TempView","description":"This step pulls an existing TempView from this session into a new DataFrame","type":"Pipeline","category":"Query","params":[{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to use"}],"engineMeta":{"spark":"QuerySteps.tempViewToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"648f27aa-6e3b-44ed-a093-bc284783731b","displayName":"Create a TempView from a DataFrame Query","description":"This step runs a SQL statement against an existing DataFrame from this session and returns a new TempView","type":"Pipeline","category":"Query","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The dataframe to query"},{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"},{"type":"text","name":"inputViewName","required":true,"parameterType":"String","description":"The name to use when creating the view representing the input dataframe (same name used in query)"},{"type":"text","name":"outputViewName","required":false,"parameterType":"String","description":"The name of the view to create (optional, random name will be created if not provided)"}],"engineMeta":{"spark":"QuerySteps.dataFrameQueryToTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"String"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"dfb8a387-6245-4b1c-ae6c-94067eb83962","displayName":"Create a DataFrame from a DataFrame Query","description":"This step runs a SQL statement against an existing DataFrame from this session and returns a new DataFrame","type":"Pipeline","category":"Query","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The dataframe to query"},{"type":"script","name":"query","required":true,"language":"sql","parameterType":"String","description":"The query to run (all tables referenced must exist as TempViews created in this session)"},{"type":"text","name":"variableMap","required":false,"parameterType":"Map[String,String]","description":"The key/value pairs to be used in variable replacement in the query"},{"type":"text","name":"inputViewName","required":true,"parameterType":"String","description":"The name to use when creating the view representing the input dataframe (same name used in query)"}],"engineMeta":{"spark":"QuerySteps.dataFrameQueryToDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"c88de095-14e0-4c67-8537-0325127e2bd2","displayName":"Cache an exising TempView","description":"This step will cache an existing TempView","type":"Pipeline","category":"Query","params":[{"type":"text","name":"viewName","required":false,"parameterType":"String","description":"The name of the view to cache"}],"engineMeta":{"spark":"QuerySteps.cacheTempView","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"0342654c-2722-56fe-ba22-e342169545af","displayName":"Copy (auto buffering)","description":"Copy the contents of the source path to the destination path. This function will call connect on both FileManagers.","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"c40169a3-1e77-51ab-9e0a-3f24fb98beef","displayName":"Copy (basic buffering)","description":"Copy the contents of the source path to the destination path using buffer sizes. This function will call connect on both FileManagers.","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to"},{"type":"text","name":"inputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for reading data during copy"},{"type":"text","name":"outputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for writing data during copy"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"f5a24db0-e91b-5c88-8e67-ab5cff09c883","displayName":"Copy (advanced buffering)","description":"Copy the contents of the source path to the destination path using full buffer sizes. This function will call connect on both FileManagers.","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to copy from"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to copy to"},{"type":"text","name":"inputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for reading data during copy"},{"type":"text","name":"outputBufferSize","required":true,"parameterType":"Int","description":"The size of the buffer to use for writing data during copy"},{"type":"text","name":"copyBufferSize","required":true,"parameterType":"Int","description":"The intermediate buffer size to use during copy"}],"engineMeta":{"spark":"FileManagerSteps.copy","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.steps.CopyResults"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"1af68ab5-a3fe-4afb-b5fa-34e52f7c77f5","displayName":"Compare File Sizes","description":"Compare the file sizes of the source and destination paths","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"srcFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The source FileManager"},{"type":"text","name":"srcPath","required":true,"parameterType":"String","description":"The path to the source"},{"type":"text","name":"destFS","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The destination FileManager"},{"type":"text","name":"destPath","required":true,"parameterType":"String","description":"The path to th destination"}],"engineMeta":{"spark":"FileManagerSteps.compareFileSizes","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Int"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"bf2c4df8-a215-480b-87d8-586984e04189","displayName":"Delete (file)","description":"Delete a file","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The FileManager"},{"type":"text","name":"path","required":true,"parameterType":"String","description":"The path to the file being deleted"}],"engineMeta":{"spark":"FileManagerSteps.deleteFile","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"Boolean"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"3d1e8519-690c-55f0-bd05-1e7b97fb6633","displayName":"Disconnect a FileManager","description":"Disconnects a FileManager from the underlying file system","type":"Pipeline","category":"FileManager","params":[{"type":"text","name":"fileManager","required":true,"parameterType":"com.acxiom.pipeline.fs.FileManager","description":"The file manager to disconnect"}],"engineMeta":{"spark":"FileManagerSteps.disconnectFileManager","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"259a880a-3e12-4843-9f02-2cfc2a05f576","displayName":"Create a FileManager","description":"Creates a FileManager using the provided FileConnector","type":"Pipeline","category":"Connectors","params":[{"type":"text","name":"fileConnector","required":true,"parameterType":"com.acxiom.pipeline.connectors.FileConnector","description":"The FileConnector to use to create the FileManager implementation"}],"engineMeta":{"spark":"FileManagerSteps.getFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.FileManager"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"9d467cb0-8b3d-40a0-9ccd-9cf8c5b6cb38","displayName":"Create SFTP FileManager","description":"Simple function to generate the SFTPFileManager for the remote SFTP file system","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"hostName","required":true,"parameterType":"String","description":"The name of the host to connect"},{"type":"text","name":"username","required":false,"parameterType":"String","description":"The username used for connection"},{"type":"text","name":"password","required":false,"parameterType":"String","description":"The password used for connection"},{"type":"integer","name":"port","required":false,"parameterType":"Int","description":"The optional port if other than 22"},{"type":"boolean","name":"strictHostChecking","required":false,"parameterType":"Boolean","description":"Option to automatically add keys to the known_hosts file. Default is false."}],"engineMeta":{"spark":"SFTPSteps.createFileManager","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.fs.SFTPFileManager"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"5c4d2d01-da85-4e2e-a551-f5a65f83653a","displayName":"Set Spark Local Property","description":"Set a property on the spark context.","type":"Pipeline","category":"Spark","params":[{"type":"text","name":"key","required":true,"parameterType":"String","description":"The name of the property to set"},{"type":"text","name":"value","required":true,"parameterType":"Any","description":"The value to set"}],"engineMeta":{"spark":"SparkConfigurationSteps.setLocalProperty","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"0b86b314-2657-4392-927c-e555af56b415","displayName":"Set Spark Local Properties","description":"Set each property on the spark context.","type":"Pipeline","category":"Spark","params":[{"type":"text","name":"properties","required":true,"parameterType":"Map[String,Any]","description":"Map representing local properties to set"},{"type":"text","name":"keySeparator","required":false,"defaultValue":"__","parameterType":"String","description":"String that will be replaced with a period character"}],"engineMeta":{"spark":"SparkConfigurationSteps.setLocalProperties","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"c8c82365-e078-4a2a-99b8-0c0e20d8102d","displayName":"Set Hadoop Configuration Properties","description":"Set each property on the hadoop configuration.","type":"Pipeline","category":"Spark","params":[{"type":"text","name":"properties","required":true,"parameterType":"Map[String,Any]","description":"Map representing local properties to set"},{"type":"text","name":"keySeparator","required":false,"defaultValue":"__","parameterType":"String","description":"String that will be replaced with a period character"}],"engineMeta":{"spark":"SparkConfigurationSteps.setHadoopConfigurationProperties","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"ea7ea3e0-d1c2-40a2-b2b7-3488489509ca","displayName":"Set Hadoop Configuration Property","description":"Set a property on the hadoop configuration.","type":"Pipeline","category":"Spark","params":[{"type":"text","name":"key","required":true,"parameterType":"String","description":"The name of the property to set"},{"type":"text","name":"value","required":true,"parameterType":"Any","description":"The value to set"}],"engineMeta":{"spark":"SparkConfigurationSteps.setHadoopConfigurationProperty","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"b7373f02-4d1e-44cf-a9c9-315a5c1ccecc","displayName":"Set Job Group","description":"Set the current thread\\'s group id and description that will be associated with any jobs.","type":"Pipeline","category":"Spark","params":[{"type":"text","name":"groupId","required":true,"parameterType":"String","description":"The name of the group"},{"type":"text","name":"description","required":true,"parameterType":"String","description":"Description of the job group"},{"type":"boolean","name":"interruptOnCancel","required":false,"defaultValue":"false","parameterType":"Boolean","description":"When true, will trigger Thread.interrupt getting called on executor threads"}],"engineMeta":{"spark":"SparkConfigurationSteps.setJobGroup","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"7394ff4d-f74d-4c9f-a55c-e0fd398fa264","displayName":"Clear Job Group","description":"Clear the current thread\\'s job group","type":"Pipeline","category":"Spark","params":[],"engineMeta":{"spark":"SparkConfigurationSteps.clearJobGroup","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"22fcc0e7-0190-461c-a999-9116b77d5919","displayName":"Build a DataFrameReader Object","description":"This step will build a DataFrameReader object that can be used to read a file into a dataframe","type":"Pipeline","category":"InputOutput","params":[{"type":"object","name":"dataFrameReaderOptions","required":true,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The options to use when loading the DataFrameReader"}],"engineMeta":{"spark":"DataFrameSteps.getDataFrameReader","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrameReader"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"66a451c8-ffbd-4481-9c37-71777c3a240f","displayName":"Load Using DataFrameReader","description":"This step will load a DataFrame given a dataFrameReader.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrameReader","required":true,"parameterType":"org.apache.spark.sql.DataFrameReader","description":"The DataFrameReader to use when creating the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.load","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"d7cf27e6-9ca5-4a73-a1b3-d007499f235f","displayName":"Load DataFrame","description":"This step will load a DataFrame given a DataFrameReaderOptions object.","type":"Pipeline","category":"InputOutput","params":[{"type":"object","name":"dataFrameReaderOptions","required":true,"className":"com.acxiom.pipeline.steps.DataFrameReaderOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameReaderOptions","description":"The DataFrameReaderOptions to use when creating the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.loadDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrame"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"8a00dcf8-e6a9-4833-871e-c1f3397ab378","displayName":"Build a DataFrameWriter Object","description":"This step will build a DataFrameWriter object that can be used to write a file into a dataframe","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to use when creating the DataFrameWriter"},{"type":"object","name":"options","required":true,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The DataFrameWriterOptions to use when writing the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.getDataFrameWriter","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.DataFrameWriter[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"9aa6ae9f-cbeb-4b36-ba6a-02eee0a46558","displayName":"Save Using DataFrameWriter","description":"This step will save a DataFrame given a dataFrameWriter[Row].","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrameWriter","required":true,"parameterType":"org.apache.spark.sql.DataFrameWriter[_]","description":"The DataFrameWriter to use when saving"}],"engineMeta":{"spark":"DataFrameSteps.save","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"e5ac3671-ee10-4d4e-8206-fec7effdf7b9","displayName":"Save DataFrame","description":"This step will save a DataFrame given a DataFrameWriterOptions object.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[_]","description":"The DataFrame to save"},{"type":"object","name":"dataFrameWriterOptions","required":true,"className":"com.acxiom.pipeline.steps.DataFrameWriterOptions","parameterType":"com.acxiom.pipeline.steps.DataFrameWriterOptions","description":"The DataFrameWriterOptions to use for saving"}],"engineMeta":{"spark":"DataFrameSteps.saveDataFrame","pkg":"com.acxiom.pipeline.steps"},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"fa05a970-476d-4617-be4d-950cfa65f2f8","displayName":"Persist DataFrame","description":"Persist a DataFrame to provided storage level.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to persist"},{"type":"text","name":"storageLevel","required":false,"parameterType":"String","description":"The optional storage mechanism to use when persisting the DataFrame"}],"engineMeta":{"spark":"DataFrameSteps.persistDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"e6fe074e-a1fa-476f-9569-d37295062186","displayName":"Unpersist DataFrame","description":"Unpersist a DataFrame.","type":"Pipeline","category":"InputOutput","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to unpersist"},{"type":"boolean","name":"blocking","required":false,"parameterType":"Boolean","description":"Optional flag to indicate whether to block while unpersisting"}],"engineMeta":{"spark":"DataFrameSteps.unpersistDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"71323226-bcfd-4fa1-bf9e-24e455e41144","displayName":"RepartitionDataFrame","description":"Repartition a DataFrame","type":"Pipeline","category":"Transforms","params":[{"type":"text","name":"dataFrame","required":true,"parameterType":"org.apache.spark.sql.Dataset[T]","description":"The DataFrame to repartition"},{"type":"text","name":"partitions","required":true,"parameterType":"Int","description":"The number of partitions to use"},{"type":"boolean","name":"rangePartition","required":false,"parameterType":"Boolean","description":"Flag indicating whether to repartition by range. This takes precedent over the shuffle flag"},{"type":"boolean","name":"shuffle","required":false,"parameterType":"Boolean","description":"Flag indicating whether to perform a normal partition"},{"type":"text","name":"partitionExpressions","required":false,"parameterType":"List[String]","description":"The partition expressions to use"}],"engineMeta":{"spark":"DataFrameSteps.repartitionDataFrame","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"org.apache.spark.sql.Dataset[T]"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"5e0358a0-d567-5508-af61-c35a69286e4e","displayName":"Javascript Step","description":"Executes a script and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"javascript","parameterType":"String","description":"Javascript to execute"}],"engineMeta":{"spark":"JavascriptSteps.processScript","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"570c9a80-8bd1-5f0c-9ae0-605921fe51e2","displayName":"Javascript Step with single object provided","description":"Executes a script with single object provided and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"javascript","parameterType":"String","description":"Javascript script to execute"},{"type":"text","name":"value","required":true,"parameterType":"Any","description":"Value to bind to the script"}],"engineMeta":{"spark":"JavascriptSteps.processScriptWithValue","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]},{"id":"f92d4816-3c62-4c29-b420-f00994bfcd86","displayName":"Javascript Step with map of objects provided","description":"Executes a script with map of objects provided and returns the result","type":"Pipeline","category":"Scripting","params":[{"type":"script","name":"script","required":true,"language":"javascript","parameterType":"String"},{"type":"text","name":"values","required":true,"parameterType":"Map[String,Any]","description":"Map of name/value pairs to bind to the script"},{"type":"boolean","name":"unwrapOptions","required":false,"parameterType":"Boolean","description":"Flag to control option unwrapping behavior"}],"engineMeta":{"spark":"JavascriptSteps.processScriptWithValues","pkg":"com.acxiom.pipeline.steps","results":{"primaryType":"com.acxiom.pipeline.PipelineStepResponse"}},"tags":["metalus-common_2.11-spark_2.4-1.8.3.jar"]}],"pkgObjs":[{"id":"com.acxiom.pipeline.steps.JDBCDataFrameReaderOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"JDBC Data Frame Reader Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"url\":{\"type\":\"string\"},\"table\":{\"type\":\"string\"},\"predicates\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"readerOptions\":{\"$ref\":\"#/definitions/DataFrameReaderOptions\"}},\"definitions\":{\"DataFrameReaderOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}},\"Schema\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}}},\"Attribute\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"name\":{\"type\":\"string\"},\"dataType\":{\"$ref\":\"#/definitions/AttributeType\"}}},\"AttributeType\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"baseType\":{\"type\":\"string\"},\"valueType\":{\"$ref\":\"#/definitions/AttributeType\"},\"nameType\":{\"$ref\":\"#/definitions/AttributeType\"},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}}}}"},{"id":"com.acxiom.pipeline.steps.DataFrameWriterOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Data Frame Writer Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"saveMode\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"bucketingOptions\":{\"$ref\":\"#/definitions/BucketingOptions\"},\"partitionBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"sortBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}},\"definitions\":{\"BucketingOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"numBuckets\":{\"type\":\"integer\"},\"columns\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}},\"required\":[\"numBuckets\"]}}}","template":"{\"form\":[{\"type\":\"select\",\"key\":\"format\",\"templateOptions\":{\"label\":\"Format\",\"placeholder\":\"\",\"valueProp\":\"value\",\"options\":[{\"value\":\"csv\",\"name\":\"CSV\"},{\"value\":\"json\",\"name\":\"JSON\"},{\"value\":\"parquet\",\"name\":\"Parquet\"},{\"value\":\"orc\",\"name\":\"Orc\"},{\"value\":\"text\",\"name\":\"Text\"}],\"labelProp\":\"name\",\"focus\":false,\"_flatOptions\":true,\"disabled\":false}},{\"key\":\"options.encoding\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Encoding\",\"placeholder\":\"\",\"focus\":false},\"expressionProperties\":{\"templateOptions.disabled\":\"!model.format\"}},{\"key\":\"saveMode\",\"type\":\"select\",\"defaultValue\":false,\"templateOptions\":{\"label\":\"Save Mode\",\"placeholder\":\"\",\"valueProp\":\"value\",\"options\":[{\"value\":\"OVERWRITE\",\"name\":\"Overwrite\"},{\"value\":\"append\",\"name\":\"Append\"},{\"value\":\"ignore\",\"name\":\"Ignore\"},{\"value\":\"error\",\"name\":\"Error\"},{\"value\":\"errorifexists\",\"name\":\"Error If Exists\"}],\"labelProp\":\"name\",\"focus\":false,\"_flatOptions\":true,\"disabled\":false}},{\"key\":\"partitionBy\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Partition By Columns\",\"placeholder\":\"Add column names to use during partitioning\"}},{\"key\":\"sortBy\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Sort By Columns\",\"placeholder\":\"Add column names to use during sorting\"}},{\"key\":\"bucketingOptions\",\"wrappers\":[\"panel\"],\"templateOptions\":{\"label\":\"Bucketing Options\"},\"fieldGroup\":[{\"key\":\"numBuckets\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Number of Buckets\",\"type\":\"number\"}},{\"key\":\"columns\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Bucket Columns\",\"placeholder\":\"Add column names to use during bucketing\"}}]},{\"key\":\"options.sep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.lineSep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Line Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'text' || model.format === 'json' ? false : true\"},{\"key\":\"options.escapeQuotes\",\"hideExpression\":\"model.format === 'csv' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Escape Quotes?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.quote\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Quote\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.escape\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Escape Character\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"}]}"},{"id":"com.acxiom.pipeline.steps.JDBCDataFrameWriterOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"JDBC Data Frame Writer Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"url\":{\"type\":\"string\"},\"table\":{\"type\":\"string\"},\"writerOptions\":{\"$ref\":\"#/definitions/DataFrameWriterOptions\"}},\"definitions\":{\"DataFrameWriterOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"saveMode\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"bucketingOptions\":{\"$ref\":\"#/definitions/BucketingOptions\"},\"partitionBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"sortBy\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}}},\"BucketingOptions\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"numBuckets\":{\"type\":\"integer\"},\"columns\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}},\"required\":[\"numBuckets\"]}}}"},{"id":"com.acxiom.pipeline.steps.Transformations","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Transformations\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"columnDetails\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/ColumnDetails\"}},\"filter\":{\"type\":\"string\"},\"standardizeColumnNames\":{}},\"definitions\":{\"ColumnDetails\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"outputField\":{\"type\":\"string\"},\"inputAliases\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"expression\":{\"type\":\"string\"}}}}}","template":"{\"form\":[{\"key\":\"filter\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Filter\"}},{\"key\":\"standardizeColumnNames\",\"type\":\"checkbox\",\"defaultValue\":false,\"templateOptions\":{\"floatLabel\":\"always\",\"align\":\"start\",\"label\":\"Standardize Column Names?\",\"hideFieldUnderline\":true,\"color\":\"accent\",\"placeholder\":\"\",\"focus\":false,\"hideLabel\":true,\"disabled\":false,\"indeterminate\":true}},{\"fieldArray\":{\"fieldGroup\":[{\"key\":\"outputField\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Output Field\"}},{\"key\":\"inputAliases\",\"type\":\"stringArray\",\"templateOptions\":{\"label\":\"Input Alias\"}},{\"key\":\"expression\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Expression (Optional)\"}}]},\"key\":\"columnDetails\",\"wrappers\":[\"panel\"],\"type\":\"repeat\",\"templateOptions\":{\"label\":\"Column Details\"}}]}"},{"id":"com.acxiom.pipeline.steps.DataFrameReaderOptions","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Data Frame Reader Options\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"format\":{\"type\":\"string\"},\"options\":{\"type\":\"object\",\"additionalProperties\":{\"type\":\"string\"}},\"schema\":{\"$ref\":\"#/definitions/Schema\"}},\"definitions\":{\"Schema\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}}},\"Attribute\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"name\":{\"type\":\"string\"},\"dataType\":{\"$ref\":\"#/definitions/AttributeType\"}}},\"AttributeType\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"baseType\":{\"type\":\"string\"},\"valueType\":{\"$ref\":\"#/definitions/AttributeType\"},\"nameType\":{\"$ref\":\"#/definitions/AttributeType\"},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}}}}","template":"{\"form\":[{\"type\":\"select\",\"key\":\"format\",\"templateOptions\":{\"label\":\"Format\",\"placeholder\":\"\",\"valueProp\":\"value\",\"options\":[{\"value\":\"csv\",\"name\":\"CSV\"},{\"value\":\"json\",\"name\":\"JSON\"},{\"value\":\"parquet\",\"name\":\"Parquet\"},{\"value\":\"orc\",\"name\":\"Orc\"},{\"value\":\"text\",\"name\":\"Text\"}],\"labelProp\":\"name\",\"focus\":false,\"_flatOptions\":true,\"disabled\":false}},{\"key\":\"options.encoding\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Encoding\",\"placeholder\":\"\",\"focus\":false},\"expressionProperties\":{\"templateOptions.disabled\":\"!model.format\"}},{\"key\":\"options.multiLine\",\"hideExpression\":\"model.format === 'csv' || model.format === 'json' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Multiline?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.header\",\"hideExpression\":\"model.format === 'csv' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Skip Header?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.sep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.lineSep\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Line Separator\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'text' || model.format === 'json' ? false : true\"},{\"key\":\"options.quote\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Quote\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.escape\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Field Escape Character\",\"placeholder\":\"\",\"focus\":false},\"hideExpression\":\"model.format === 'csv' ? false : true\"},{\"key\":\"options.primitivesAsString\",\"hideExpression\":\"model.format === 'json' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Primitive As String?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"key\":\"options.inferSchema\",\"hideExpression\":\"model.format === 'csv' ? false : true\",\"type\":\"select\",\"templateOptions\":{\"label\":\"Infer Schema?\",\"options\":[{\"value\":\"true\",\"name\":\"True\"},{\"value\":\"false\",\"name\":\"False\"}],\"valueProp\":\"value\",\"labelProp\":\"name\"},\"defaultValue\":\"false\"},{\"expressionProperties\":{\"templateOptions.disabled\":\"model.options.inferSchema || model.format === 'json' ? false : true\"},\"key\":\"options.samplingRatio\",\"hideExpression\":\"model.format === 'csv' || model.format === 'json' ? false : true\",\"type\":\"input\",\"templateOptions\":{\"label\":\"Sampling Ration\",\"type\":\"number\"}}]}"},{"id":"com.acxiom.pipeline.steps.Schema","schema":"{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Schema\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}},\"definitions\":{\"Attribute\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"name\":{\"type\":\"string\"},\"dataType\":{\"$ref\":\"#/definitions/AttributeType\"}}},\"AttributeType\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"baseType\":{\"type\":\"string\"},\"valueType\":{\"$ref\":\"#/definitions/AttributeType\"},\"nameType\":{\"$ref\":\"#/definitions/AttributeType\"},\"schema\":{\"$ref\":\"#/definitions/Schema\"}}},\"Schema\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"attributes\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/definitions/Attribute\"}}}}}}"}]} diff --git a/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineStepMapper.scala b/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineStepMapper.scala index 5a968b9d..a3d5b733 100644 --- a/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineStepMapper.scala +++ b/metalus-core/src/main/scala/com/acxiom/pipeline/PipelineStepMapper.scala @@ -218,8 +218,7 @@ trait PipelineStepMapper { def mapParameter(parameter: Parameter, pipelineContext: PipelineContext): Any = { // Get the value/defaultValue for this parameter val value = getParamValue(parameter) - val returnValue = if (value.isDefined) { - removeOptions(value) match { + val returnValue = value.map(removeOptions).flatMap { case s: String => parameter.`type`.getOrElse("none").toLowerCase match { case "script" => @@ -239,13 +238,11 @@ trait PipelineStepMapper { case b: Boolean => Some(b) case i: Int => Some(i) case i: BigInt => Some(i.toInt) + case d: Double => Some(d) case l: List[_] => handleListParameter(l, parameter, pipelineContext) case m: Map[_, _] => handleMapParameter(m, parameter, pipelineContext) case t => // Handle other types - This function may need to be reworked to support this so that it can be overridden throw new RuntimeException(s"Unsupported value type ${t.getClass} for ${parameter.name.getOrElse("unknown")}!") - } - } else { - None } // use the first valid (non-empty) value found diff --git a/metalus-core/src/main/scala/com/acxiom/pipeline/applications/ApplicationUtils.scala b/metalus-core/src/main/scala/com/acxiom/pipeline/applications/ApplicationUtils.scala index 97f61789..0ce62b70 100644 --- a/metalus-core/src/main/scala/com/acxiom/pipeline/applications/ApplicationUtils.scala +++ b/metalus-core/src/main/scala/com/acxiom/pipeline/applications/ApplicationUtils.scala @@ -65,6 +65,7 @@ object ApplicationUtils { * @param pipelineListener An optional PipelineListener. This may be overridden by the application. * @return An execution plan. */ + //noinspection ScalaStyle def createExecutionPlan(application: Application, globals: Option[Map[String, Any]], sparkConf: SparkConf, pipelineListener: PipelineListener = PipelineListener(), applicationTriggers: ApplicationTriggers = ApplicationTriggers(), @@ -77,18 +78,23 @@ object ApplicationUtils { logger.info(s"setting parquet dictionary enabled to ${applicationTriggers.parquetDictionaryEnabled.toString}") sparkSession.sparkContext.hadoopConfiguration.set("parquet.enable.dictionary", applicationTriggers.parquetDictionaryEnabled.toString) implicit val formats: Formats = getJson4sFormats(application.json4sSerializers) + val globalStepMapper = generateStepMapper(application.stepMapper, Some(PipelineStepMapper()), + applicationTriggers.validateArgumentTypes, credentialProvider) val rootGlobals = globals.getOrElse(Map[String, Any]()) // Create the default globals - val defaultGlobals = generateGlobals(application.globals, rootGlobals, Some(rootGlobals)) val globalListener = generatePipelineListener(application.pipelineListener, Some(pipelineListener), applicationTriggers.validateArgumentTypes, credentialProvider) val globalSecurityManager = generateSecurityManager(application.securityManager, Some(PipelineSecurityManager()), applicationTriggers.validateArgumentTypes, credentialProvider) - val globalStepMapper = generateStepMapper(application.stepMapper, Some(PipelineStepMapper()), - applicationTriggers.validateArgumentTypes, credentialProvider) val globalPipelineParameters = generatePipelineParameters(application.pipelineParameters, Some(PipelineParameters())) val pipelineManager = generatePipelineManager(application.pipelineManager, Some(PipelineManager(application.pipelines.getOrElse(List[DefaultPipeline]()))), applicationTriggers.validateArgumentTypes, credentialProvider).get + val initialContext = PipelineContext(Some(sparkConf), Some(sparkSession), Some(rootGlobals), globalSecurityManager.get, + globalPipelineParameters.get, application.stepPackages, globalStepMapper.get, globalListener, + Some(sparkSession.sparkContext.collectionAccumulator[PipelineStepMessage]("stepMessages")), + ExecutionAudit("root", AuditType.EXECUTION, Map[String, Any](), System.currentTimeMillis()), pipelineManager, + credentialProvider, Some(formats)) + val defaultGlobals = generateGlobals(application.globals, rootGlobals , Some(rootGlobals), initialContext) generateSparkListeners(application.sparkListeners, applicationTriggers.validateArgumentTypes, credentialProvider).getOrElse(List()).foreach(sparkSession.sparkContext.addSparkListener) addSparkListener(globalListener, sparkSession) @@ -101,18 +107,17 @@ object ApplicationUtils { } generateSparkListeners(execution.sparkListeners, applicationTriggers.validateArgumentTypes, credentialProvider).getOrElse(List()).foreach(sparkSession.sparkContext.addSparkListener) + val stepMapper = generateStepMapper(execution.stepMapper, globalStepMapper, applicationTriggers.validateArgumentTypes, + credentialProvider).get // Extracting pipelines - val ctx = PipelineContext(Some(sparkConf), - Some(sparkSession), - generateGlobals(execution.globals, rootGlobals, defaultGlobals, execution.mergeGlobals.getOrElse(false)), - generateSecurityManager(execution.securityManager, globalSecurityManager, + val ctx = initialContext.copy( + globals = generateGlobals(execution.globals, rootGlobals, defaultGlobals, initialContext, execution.mergeGlobals.getOrElse(false)), + security = generateSecurityManager(execution.securityManager, globalSecurityManager, applicationTriggers.validateArgumentTypes, credentialProvider).get, - generatePipelineParameters(execution.pipelineParameters, globalPipelineParameters).get, application.stepPackages, - generateStepMapper(execution.stepMapper, globalStepMapper, applicationTriggers.validateArgumentTypes, - credentialProvider).get, pipelineListener, - Some(sparkSession.sparkContext.collectionAccumulator[PipelineStepMessage]("stepMessages")), - ExecutionAudit("root", AuditType.EXECUTION, Map[String, Any](), System.currentTimeMillis()), - pipelineManager, credentialProvider, Some(formats)) + parameters = generatePipelineParameters(execution.pipelineParameters, globalPipelineParameters).get, + parameterMapper = stepMapper, + pipelineListener = pipelineListener + ) PipelineExecution(execution.id.getOrElse(""), generatePipelines(execution, application, pipelineManager), execution.initialPipelineId, ctx, execution.parents) }) @@ -132,10 +137,12 @@ object ApplicationUtils { execution: Execution, pipelineExecution: PipelineExecution): PipelineExecution = { implicit val formats: Formats = getJson4sFormats(application.json4sSerializers) - val defaultGlobals = generateGlobals(application.globals, rootGlobals.get, rootGlobals) + val initialContext = pipelineExecution.pipelineContext.copy(globals = rootGlobals) + val defaultGlobals = generateGlobals(application.globals, rootGlobals.get, rootGlobals, initialContext) val globalPipelineParameters = generatePipelineParameters(application.pipelineParameters, Some(PipelineParameters())) val ctx = pipelineExecution.pipelineContext - .copy(globals = generateGlobals(execution.globals, rootGlobals.get, defaultGlobals, execution.mergeGlobals.getOrElse(false))) + .copy(globals = generateGlobals(execution.globals, rootGlobals.get, defaultGlobals, + initialContext, execution.mergeGlobals.getOrElse(false))) .copy(parameters = generatePipelineParameters(execution.pipelineParameters, globalPipelineParameters).get) pipelineExecution.asInstanceOf[DefaultPipelineExecution].copy(pipelineContext = ctx) } @@ -266,18 +273,25 @@ object ApplicationUtils { private def generateGlobals(globals: Option[Map[String, Any]], rootGlobals: Map[String, Any], defaultGlobals: Option[Map[String, Any]], + pipelineContext: PipelineContext, merge: Boolean = false)(implicit formats: Formats): Option[Map[String, Any]] = { - if (globals.isEmpty) { - defaultGlobals - } else { - val baseGlobals = globals.get - val result = baseGlobals.foldLeft(rootGlobals)((rootMap, entry) => parseValue(rootMap, entry._1, entry._2)) - Some(if (merge) { + globals.map { baseGlobals => + val result = rootGlobals ++ baseGlobals.map{ + case (key, m: Map[String, Any]) if m.contains("className") => + key -> Parameter(Some("object"), Some(key), value = m.get("object"), className = m.get("className").map(_.toString)) + case (key, l: List[Any]) => key -> Parameter(Some("list"), Some(key), value = Some(l)) + case (key, value) => key -> Parameter(Some("text"), Some(key), value = Some(value)) + }.map{ + case ("GlobalLinks", p) => "GlobalLinks" -> p.value.get // skip global links + case (key, p) => key -> pipelineContext.parameterMapper.mapParameter(p, pipelineContext) + } + // val result = baseGlobals.foldLeft(rootGlobals)((rootMap, entry) => parseValue(rootMap, entry._1, entry._2)) + if (merge) { defaultGlobals.getOrElse(Map[String, Any]()) ++ result } else { result - }) - } + } + }.orElse(defaultGlobals) } private def parseParameters(classInfo: ClassInfo, credentialProvider: Option[CredentialProvider])(implicit formats: Formats): Map[String, Any] = { diff --git a/metalus-core/src/test/scala/com/acxiom/pipeline/applications/ApplicationTests.scala b/metalus-core/src/test/scala/com/acxiom/pipeline/applications/ApplicationTests.scala index ffbc0226..4b3716f7 100644 --- a/metalus-core/src/test/scala/com/acxiom/pipeline/applications/ApplicationTests.scala +++ b/metalus-core/src/test/scala/com/acxiom/pipeline/applications/ApplicationTests.scala @@ -399,7 +399,7 @@ class ApplicationTests extends FunSpec with BeforeAndAfterAll with Suite { assert(globals.contains("rootLogLevel")) assert(globals.contains("rootLogLevel")) assert(globals.contains("number")) - assert(globals("number").asInstanceOf[BigInt] == 5) + assert(globals("number").asInstanceOf[Int] == 5) assert(globals.contains("float")) assert(globals("float").asInstanceOf[Double] == 1.5) assert(globals.contains("string")) @@ -452,7 +452,7 @@ class ApplicationTests extends FunSpec with BeforeAndAfterAll with Suite { assert(globals.contains("rootLogLevel")) assert(globals.contains("rootLogLevel")) assert(globals.contains("number")) - assert(globals("number").asInstanceOf[BigInt] == 2) + assert(globals("number").asInstanceOf[Int] == 2) assert(globals.contains("float")) assert(globals("float").asInstanceOf[Double] == 3.5) assert(globals.contains("string")) @@ -492,7 +492,7 @@ class ApplicationTests extends FunSpec with BeforeAndAfterAll with Suite { assert(globals1.contains("rootLogLevel")) assert(globals1.contains("rootLogLevel")) assert(globals1.contains("number")) - assert(globals1("number").asInstanceOf[BigInt] == 1) + assert(globals1("number").asInstanceOf[Int] == 1) assert(globals1.contains("float")) assert(globals1("float").asInstanceOf[Double] == 1.5) assert(globals1.contains("string")) @@ -565,7 +565,7 @@ class ApplicationTests extends FunSpec with BeforeAndAfterAll with Suite { assert(globals.contains("rootLogLevel")) assert(globals.contains("rootLogLevel")) assert(globals.contains("number")) - assert(globals("number").asInstanceOf[BigInt] == 2) + assert(globals("number").asInstanceOf[Int] == 2) assert(globals.contains("float")) assert(globals("float").asInstanceOf[Double] == 3.5) assert(globals.contains("string"))