From eb38fa6be7fb919d04ac4865dd33cd670fe8f0d0 Mon Sep 17 00:00:00 2001 From: minyk Date: Tue, 21 May 2019 12:37:41 +0900 Subject: [PATCH] update project to present. --- build.sbt | 61 ++++++++++++++++++++++++++++++-- project/BuildSettings.scala | 54 ---------------------------- project/Dependencies.scala | 26 -------------- project/ProjectBuild.scala | 27 -------------- project/build.properties | 6 ++-- project/plugins.sbt | 12 ++----- src/main/resources/parameters.sh | 1 + 7 files changed, 65 insertions(+), 122 deletions(-) delete mode 100644 project/BuildSettings.scala delete mode 100644 project/Dependencies.scala delete mode 100644 project/ProjectBuild.scala diff --git a/build.sbt b/build.sbt index d94e386..e38da75 100644 --- a/build.sbt +++ b/build.sbt @@ -1,3 +1,60 @@ -net.virtualvoid.sbt.graph.Plugin.graphSettings +//net.virtualvoid.sbt.graph.Plugin.graphSettings -scalaVersion := "2.10.5" +scalaVersion := "2.11.11" + +resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns) +resolvers += "Typesafe Repository" at "http://repo.typesafe.com/typesafe/releases/" +resolvers += "sonatype-releases" at "https://oss.sonatype.org/content/repositories/releases/" +resolvers += "Akka Repository" at "http://repo.akka.io/releases/" +resolvers += "Spray Repository" at "http://repo.spray.cc/" +resolvers += "Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases" + +libraryDependencies ++= Seq( + "org.apache.spark" %% "spark-core" % "2.4.0" % "provided", + "org.apache.hadoop" % "hadoop-client" % "2.9.2" % "provided", + "com.google.guava" % "guava" % "14.0.1", + "org.specs2" %% "specs2" % "3.7" % "test" +) + +// lazy val commonSettings = Seq( +// organization := "com.nexr", +// version := "0.2", +// description := "Simple terasort.", +// scalaVersion := "2.11.11", +// scalacOptions := Seq("-deprecation", "-encoding", "utf8"), +// resolvers +// ) + +// lazy val app = (project in file(".")). +// settings(commonSettings: _*). +// settings( +// assemblyJarName in assembly := { +// name.value + "-" + version.value + ".jar" +// }, +// +// // Drop these jars +// //excludedJars in assembly := (fullClasspath in assembly) { (cp) => +// // val excludes = Set( +// // "jsp-api-2.1-6.1.14.jar", +// // "jsp-2.1-6.1.14.jar", +// // "jasper-compiler-5.5.12.jar", +// // "commons-beanutils-core-1.8.0.jar", +// // "commons-beanutils-1.7.0.jar", +// // "servlet-api-2.5-20081211.jar", +// // "servlet-api-2.5.jar" +// // ) +// // cp filter { jar => excludes(jar.data.getName) } +// //}, +// +// //mergeStrategy in assembly := (mergeStrategy in assembly) { +// // // case "project.clj" => MergeStrategy.discard // Leiningen build files +// // case x if x.startsWith("META-INF") => MergeStrategy.discard // Bumf +// // case x if x.startsWith("hadoop") && x.contains("2.9.2") => MergeStrategy.discard +// // case x if x.endsWith(".html") => MergeStrategy.discard // More bumf +// // case x if x.endsWith("package-info.class") => MergeStrategy.last +// // case PathList("com", "esotericsoftware", xs @ _*) => MergeStrategy.last // For Log$Logger.class +// // case x => +// // val oldStrategy = (mergeStrategy in assembly).value +// // oldStrategy(x) +// //} +// ) diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala deleted file mode 100644 index 412ebcb..0000000 --- a/project/BuildSettings.scala +++ /dev/null @@ -1,54 +0,0 @@ -import sbt._ -import Keys._ - -object BuildSettings { - - // Basic settings for our app - lazy val basicSettings = Seq[Setting[_]]( - organization := "com.nexr", - version := "0.1", - description := "Simple terasort.", - scalaVersion := "2.10.5", // -> 2.10.0 when Spark is ready - scalacOptions := Seq("-deprecation", "-encoding", "utf8"), - resolvers ++= Dependencies.resolutionRepos - ) - - // sbt-assembly settings for building a fat jar - import sbtassembly.Plugin._ - import AssemblyKeys._ - lazy val sbtAssemblySettings = assemblySettings ++ Seq( - - // Slightly cleaner jar name - jarName in assembly := { - name.value + "-" + version.value + ".jar" - }, - - // Drop these jars - excludedJars in assembly <<= (fullClasspath in assembly) map { cp => - val excludes = Set( - "jsp-api-2.1-6.1.14.jar", - "jsp-2.1-6.1.14.jar", - "jasper-compiler-5.5.12.jar", - "commons-beanutils-core-1.8.0.jar", - "commons-beanutils-1.7.0.jar", - "servlet-api-2.5-20081211.jar", - "servlet-api-2.5.jar" - ) - cp filter { jar => excludes(jar.data.getName) } - }, - - mergeStrategy in assembly <<= (mergeStrategy in assembly) { - (old) => { - // case "project.clj" => MergeStrategy.discard // Leiningen build files - case x if x.startsWith("META-INF") => MergeStrategy.discard // Bumf - case x if x.startsWith("hadoop") && x.contains("2.2.0") => MergeStrategy.discard - case x if x.endsWith(".html") => MergeStrategy.discard // More bumf - case x if x.endsWith("package-info.class") => MergeStrategy.last - case PathList("com", "esotericsoftware", xs @ _*) => MergeStrategy.last // For Log$Logger.class - case x => old(x) - } - } - ) - - lazy val buildSettings = basicSettings ++ sbtAssemblySettings -} \ No newline at end of file diff --git a/project/Dependencies.scala b/project/Dependencies.scala deleted file mode 100644 index 361fdf8..0000000 --- a/project/Dependencies.scala +++ /dev/null @@ -1,26 +0,0 @@ -import sbt._ - -object Dependencies { - val resolutionRepos = Seq( - "Akka Repository" at "http://repo.akka.io/releases/", - "Spray Repository" at "http://repo.spray.cc/", - "Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases" - ) - - object V { - val spark = "1.2.1" - val specs2 = "2.4.17" // -> "1.13" when we bump to Scala 2.10.0 - val guava = "14.0.1" - val hadoop = "2.6.0" - // Add versions for your additional libraries here... - } - - object Libraries { - val sparkCore = "org.apache.spark" %% "spark-core" % V.spark % "provided" - val hadoopClient = "org.apache.hadoop" % "hadoop-client" % V.hadoop % "provided" - val guava = "com.google.guava" % "guava" % V.guava - - // Scala (test only) - val specs2 = "org.specs2" %% "specs2" % V.specs2 % "test" - } -} \ No newline at end of file diff --git a/project/ProjectBuild.scala b/project/ProjectBuild.scala deleted file mode 100644 index 5470440..0000000 --- a/project/ProjectBuild.scala +++ /dev/null @@ -1,27 +0,0 @@ -import sbt._ -import Keys._ - -object ProjectBuild extends Build { - - import Dependencies._ - import BuildSettings._ - - // Configure prompt to show current project - override lazy val settings = super.settings :+ { - shellPrompt := { s => Project.extract(s).currentProject.id + " > " } - } - - // Define our project, with basic project information and library dependencies - lazy val project = Project("spark-terasort", file(".")) - .settings(net.virtualvoid.sbt.graph.Plugin.graphSettings: _*) - .settings(buildSettings: _*) - .settings( - libraryDependencies ++= Seq( - Libraries.sparkCore, - Libraries.hadoopClient, - Libraries.guava, - Libraries.specs2 - // Add your additional libraries here (comma-separated)... - ) - ) -} \ No newline at end of file diff --git a/project/build.properties b/project/build.properties index 8c2551a..7313073 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,6 +1,6 @@ # #Thu Mar 05 13:54:53 KST 2015 project.name=spark-terasort -project.version=0.1 -sbt.version=0.13.5 -build.scala.versions=2.10.5 +project.version=0.2 +sbt.version=1.2.8 +build.scala.versions=2.11.11 diff --git a/project/plugins.sbt b/project/plugins.sbt index 30e7350..86278e8 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,13 +1,5 @@ logLevel := Level.Warn -resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns) +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.9") -resolvers += "Typesafe Repository" at "http://repo.typesafe.com/typesafe/releases/" - -resolvers += "sonatype-releases" at "https://oss.sonatype.org/content/repositories/releases/" - -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2") - -addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.7.4") - -addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.6.0") \ No newline at end of file +addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2") diff --git a/src/main/resources/parameters.sh b/src/main/resources/parameters.sh index e5022c3..c2ca04d 100644 --- a/src/main/resources/parameters.sh +++ b/src/main/resources/parameters.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash HADOOP_USER=root HDFS_USER=hdfs SLEEP_BETWEEN_RUNS=60