diff --git a/.gitignore b/.gitignore index a04ff5b4..eef8c62d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ .bloop +.bsp .metals .venv .vscode +.DS_Store project target diff --git a/build.sbt b/build.sbt index dc8297a6..be98abd7 100644 --- a/build.sbt +++ b/build.sbt @@ -22,6 +22,7 @@ val s3MockVersion = "0.2.6" val scalaXmlVersion = "2.1.0" val scalaParserCombinatorsVersion = "2.2.0" val gcsConnectorVersion = "hadoop3-2.2.2" +val monovoreDeclineVersion = "2.4.1" // ----- TOOL DEPENDENCIES ----- // @@ -47,11 +48,15 @@ val scalaParserCombinatorsDependencies = Seq( val sparkDependencies = Seq( "org.apache.spark" %% "spark-core" % sparkVersion, - "org.apache.spark" %% "spark-sql" % sparkVersion, - "org.apache.spark" %% "spark-hadoop-cloud" % sparkVersion + "org.apache.spark" %% "spark-sql" % sparkVersion ) .map(_ excludeAll ("org.scala-lang.modules", "scala-xml")) .map(_.cross(CrossVersion.for3Use2_13)) + .map(_ % "provided") + +val sparkHadoopCloudDependencies = Seq("org.apache.spark" %% "spark-hadoop-cloud" % sparkVersion) + .map(_ exclude ("org.apache.hadoop", "hadoop-client-runtime")) + .map(_.cross(CrossVersion.for3Use2_13)) val catsDependencies = Seq("org.typelevel" %% "cats-core" % catsVersion) @@ -65,12 +70,15 @@ val s3MockDependencies = Seq("io.findify" %% "s3mock" % s3MockVersion % "it,test val gcsConnectorDependencies = Seq("com.google.cloud.bigdataoss" % "gcs-connector" % gcsConnectorVersion) +val monovoreDeclineDependencies = Seq("com.monovore" %% "decline" % monovoreDeclineVersion) + // ----- MODULE DEPENDENCIES ----- // val restonomerCoreDependencies = zioConfigDependencies ++ scalaXmlDependencies ++ scalaParserCombinatorsDependencies ++ + sparkDependencies ++ sttpDependencies ++ jwtDependencies ++ jsonPathDependencies ++ @@ -78,10 +86,12 @@ val restonomerCoreDependencies = wireMockDependencies ++ s3MockDependencies ++ odelayDependencies ++ - gcsConnectorDependencies + gcsConnectorDependencies ++ + monovoreDeclineDependencies val restonomerSparkUtilsDependencies = sparkDependencies ++ + sparkHadoopCloudDependencies ++ catsDependencies ++ scalaTestDependencies.map(_ % "test") @@ -112,9 +122,11 @@ lazy val restonomer = (project in file(".")) commonSettings ++ Seq( publish / skip := true, publishLocal / skip := true - ) + ), + addCommandAlias("run", "restonomer-core/run") ) .aggregate(`restonomer-core`, `restonomer-spark-utils`) + .enablePlugins(AssemblyPlugin) lazy val `restonomer-core` = project .configs(IntegrationTest) @@ -135,3 +147,26 @@ ThisBuild / credentials += Credentials( "teamclairvoyant", System.getenv("GITHUB_TOKEN") ) + +// ----- ASSEMBLY MERGE STRATEGY ----- // + +ThisBuild / assemblyMergeStrategy := { + case PathList(ps @ _*) + if ps.last endsWith "io.netty.versions.properties" => + MergeStrategy.last + case PathList(ps @ _*) + if ps.last endsWith "native-image.properties" => + MergeStrategy.last + case PathList(ps @ _*) + if ps.last endsWith "reflection-config.json" => + MergeStrategy.last + case PathList(ps @ _*) if ps.last endsWith "public-suffix-list.txt" => + MergeStrategy.concat + case PathList(ps @ _*) if ps.last endsWith "module-info.class" => + MergeStrategy.first + case PathList(ps @ _*) if ps.last endsWith "UnusedStubClass.class" => + MergeStrategy.first + case x => + val oldStrategy = (ThisBuild / assemblyMergeStrategy).value + oldStrategy(x) +} diff --git a/project/plugins.sbt b/project/plugins.sbt index 83adafa3..974869be 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1 +1,2 @@ addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0") \ No newline at end of file diff --git a/restonomer-core/src/main/scala/com/clairvoyant/restonomer/core/app/RestonomerApp.scala b/restonomer-core/src/main/scala/com/clairvoyant/restonomer/core/app/RestonomerApp.scala new file mode 100644 index 00000000..9cbb6a9f --- /dev/null +++ b/restonomer-core/src/main/scala/com/clairvoyant/restonomer/core/app/RestonomerApp.scala @@ -0,0 +1,24 @@ +package com.clairvoyant.restonomer.core.app + +import cats.implicits.* +import com.clairvoyant.restonomer.core.app.RestonomerContext +import com.monovore.decline.CommandApp +import com.monovore.decline.Opts + +object RestonomerApp + extends CommandApp( + name = "restonomer-app", + header = "Dumps the data from the REST API provided in the checkpoint", + main = { + val restonomerContextDirPathOpt: Opts[String] = Opts + .option[String](long = "restonomer_context_dir_path", help = "Full path to the restonomer context directory") + + val checkpointFilePathOpt: Opts[String] = Opts + .option[String](long = "checkpoint_file_path", help = "Relative path of the checkpoint file to be executed") + + (restonomerContextDirPathOpt, checkpointFilePathOpt).mapN { (restonomerContextDirPath, checkpointFilePath) => + val restonomerContext = RestonomerContext(restonomerContextDirPath) + restonomerContext.runCheckpoint(checkpointFilePath) + } + } + )