Switch to Travis CI for building releases

Both nightly and full releases migrate to Travis. PR validation remains on our Jenkins cluster for now. Main difference is that we don't use our artifactory as a cache, nor do we need to wipe stuff, since Travis gives us a fresh machine. We do cache the ivy/sbt cache. TODO: decide whether to publish a "mergely" (a nightly for each merge, or skip this script on merge and use a scheduled job for publishing the nightly.) Note: we don't use `travis encrypt-file` because it nukes the iv/key variables on each invocation.. too much magic Instead, I did: ``` cat /dev/urandom | head -c 10000 | openssl sha1 > ./secret openssl aes-256-cbc -pass "file:./secret" -in gpg_subkey -out admin/files/gpg_subkey.enc travis encrypt "GPG_SUBKEY_SECRET=$(cat ./secret)" ```
retronym · Mar 7, 2018 · 41e376a · 41e376a
1 parent 2ddc01b
commit 41e376a
Show file tree

Hide file tree

Showing 11 changed files with 161 additions and 31 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,19 +1,37 @@
-# opt-in to Travis's newer/faster container-based infrastructure
-sudo: false
+sudo: required # GCE VMs have better performance (will be upgrading to premium VMs soon)
 
 # this builds the spec using jekyll
 # based on http://www.paperplanes.de/2013/8/13/deploying-your-jekyll-blog-to-s3-with-travis-ci.html
-language: ruby
+
+language: scala
+jdk: openjdk8
+
+# the spec is built with jekyll
 rvm:
   - 2.2
-script: bundle exec jekyll build -s spec/ -d build/spec
+
+cache:
+  directories:
+    - $HOME/.ivy2/cache
+    - $HOME/.sbt
+
+script:
+  - (cd admin && ./init.sh)
+  - scripts/jobs/integrate/bootstrap
+  - bundle exec jekyll build -s spec/ -d build/spec
+
 install: bundle install
 
 # cat /dev/urandom | head -c 10000 | openssl sha1 > ./secret
 # openssl aes-256-cbc -pass "file:./secret" -in id_dsa_spec212_b4096 -out spec/id_dsa_travis.enc -a
 # travis encrypt "PRIV_KEY_SECRET=`cat ./secret`"
 env:
-  - secure: "TuJOUtALynPd+MV1AuMeIpVb8BUBHr7Ul7FS48XhS2PyuTRpEBkSWybYcNg3AXyzmWDAuOjUxbaNMQBvP8vvehTbIYls5H5wTGKvj0D0TNVaPIXjF8bA8KyNat9xGNzhnWm2/2BMaWpKBJWRF7Jb+zHhijMYCJEbkMtoiE5R/mY="
+  global:
+    - secure: "TuJOUtALynPd+MV1AuMeIpVb8BUBHr7Ul7FS48XhS2PyuTRpEBkSWybYcNg3AXyzmWDAuOjUxbaNMQBvP8vvehTbIYls5H5wTGKvj0D0TNVaPIXjF8bA8KyNat9xGNzhnWm2/2BMaWpKBJWRF7Jb+zHhijMYCJEbkMtoiE5R/mY="
+    - secure: "T1fxtvLTxioyXJYiC/zVYdNYsBOt+0Piw+xE04rB1pzeKahm9+G2mISdcAyqv6/vze9eIJt6jNHHpKX32/Z3Cs1/Ruha4m3k+jblj3S0SbxV6ht2ieJXLT5WoUPFRrU68KXI8wqUadXpjxeJJV53qF2FC4lhfMUsw1IwwMhdaE8=" # PRIVATE_REPO_PASS
+    - secure: "feE5A8mYNpkNQKVwCj3aXrwjVrJWh/4ENpRfFlr2HOD9ORk1GORD5Yq907WZd+dTkYK54Lh1gA+qHOCIDgJHbi9ZLU+kjzEjtYKF6lQy6Wb0LI8smTOnAA6IWVVYifiXw8d66MI2MKZb2jjGeIzy8Q00SZjLhEGjLyTeCIB88Ws=" # SONA_USER
+    - secure: "ek3As5q2tL8UBXcxSBbv4v5YgsoPD41SCzPOSu72kzfbngyxgQxrcziU5pIM+Lib9KaWex7hVVWNL38tMyDbu+0OpDv8bPjMujzlDx5I2pJUfuOJo7QRYsJE1nsXcY4cA72cCLfbRcLEkvtDAhcdLSaUOqlyQe5BY4X4fY5eoPA=" # SONA_PASS
+    - secure: "dbAvl6KEuLwZ0MVQPZihFsPzCdiLbX0EFk3so+hcfEbksrmLQ1tn4X5ZM7Wy1UDR8uN9lxngEwHch7a7lKqpugzmXMew9Wnikr9WBWbJT77Z+XJ/jHI6YuiCRpRo+nvxXGp9Ry80tSIgx5eju0J83IaJL41BWlBkvyAd7YAHORI=" # GPG_SUBKEY_SECRET
 
 # ^^^ set PRIV_KEY_SECRET to password used to encrypt spec/id_dsa_travis.enc
 
@@ -23,3 +41,8 @@ after_success:
 
 # using S3 would be simpler, but we want to upload to scala-lang.org
 # after_success: bundle exec s3_website push --headless
+
+before_cache:
+  # Cleanup the cached directories to avoid unnecessary cache updates
+  - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete
+  - find $HOME/.sbt        -name "*.lock"               -print -delete
diff --git a/admin/files/credentials-private-repo b/admin/files/credentials-private-repo
@@ -0,0 +1,4 @@
+realm=Artifactory Realm
+host=scala-ci.typesafe.com
+user=scala-ci
+password=${PRIVATE_REPO_PASS}
diff --git a/admin/files/credentials-sonatype b/admin/files/credentials-sonatype
@@ -0,0 +1,4 @@
+realm=Sonatype Nexus Repository Manager
+host=oss.sonatype.org
+user=${SONA_USER}
+password=${SONA_PASS}
diff --git a/admin/files/gpg.sbt b/admin/files/gpg.sbt
@@ -0,0 +1 @@
+addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0")
diff --git a/admin/files/gpg_subkey.enc b/admin/files/gpg_subkey.enc
diff --git a/admin/files/m2-settings.xml b/admin/files/m2-settings.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<settings>
+  <servers>
+    <server>
+      <id>sonatype-nexus</id>
+      <username>${SONA_USER}</username>
+      <password>${SONA_PASS}</password>
+    </server>
+    <server>
+      <id>private-repo</id>
+      <username>scala-ci</username>
+      <password>${PRIVATE_REPO_PASS}</password>
+    </server>
+  </servers>
+
+  <mirrors>
+    <!--
+      codehaus doesn't exist anymore. it doesn't 404, but serves up
+      bad POMs, so we disable it globally by overriding with a mirror
+      with an invalid URL.
+      this entry could be removed after the scala-ide/scala-refactoring
+      repo removes their reference to codehaus-snapshots from their pom.xml
+    -->
+    <mirror>
+      <id>codehaus-snapshots-mirror</id>
+      <name>Maven Codehaus snapshot repository</name>
+      <url>file:///codehaus-does-not-exist-anymore</url>
+      <mirrorOf>codehaus-snapshots</mirrorOf>
+    </mirror>
+  </mirrors>
+</settings>
diff --git a/admin/files/sonatype-curl b/admin/files/sonatype-curl
@@ -0,0 +1 @@
+user = ${SONA_USER}:${SONA_PASS}
diff --git a/admin/init.sh b/admin/init.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+
+sensitive() {
+  perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/credentials-private-repo > ~/.credentials-private-repo
+  perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/credentials-sonatype     > ~/.credentials-sonatype
+  perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/sonatype-curl            > ~/.sonatype-curl
+  # perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/m2-settings.xml          > ~/.m2/settings.xml  -- not needed anymore (used for ide integration?)
+
+  openssl aes-256-cbc -d -pass "pass:$GPG_SUBKEY_SECRET" -in files/gpg_subkey.enc | gpg --import
+}
+
+# directories needed by sensitive part
+# mkdir -p ~/.m2 -- not needed anymore (used for ide integration?)
+mkdir -p ~/.ssh
+
+# don't let anything escape from the sensitive part (e.g. leak environment var by echoing to log on failure)
+sensitive >/dev/null 2>&1
+
+# pgp signing doesn't work without public key??
+gpg --keyserver pgp.mit.edu --recv-keys 0xa9052b1b6d92e560
+
+# just to verify
+gpg --list-keys
+gpg --list-secret-keys
+
+mkdir -p ~/.sbt/0.13/plugins
+cp files/gpg.sbt ~/.sbt/0.13/plugins/
+
+export SBT_CMD=$(which sbt)
diff --git a/project/ScriptCommands.scala b/project/ScriptCommands.scala
@@ -6,6 +6,8 @@ import BuildSettings.autoImport._
 
 /** Custom commands for use by the Jenkins scripts. This keeps the surface area and call syntax small. */
 object ScriptCommands {
+  def env(key: String) = Option(System.getenv(key)).getOrElse("")
+
   def all = Seq(
     setupPublishCore,
     setupValidateTest,
@@ -80,7 +82,7 @@ object ScriptCommands {
       baseVersionSuffix in Global := "SPLIT",
       resolvers in Global += "scala-pr" at url,
       publishTo in Global := Some("sonatype-releases" at "https://oss.sonatype.org/service/local/staging/deploy/maven2"),
-      credentials in Global += Credentials(Path.userHome / ".credentials-sonatype"),
+      credentials in Global += Credentials("Sonatype Nexus Repository Manager", "oss.sonatype.org", env("SONA_USER"), env("SONA_PASS")),
       pgpPassphrase in Global := Some(Array.empty)
     ) ++ enableOptimizer
   }
@@ -114,7 +116,11 @@ object ScriptCommands {
   private[this] def publishTarget(url: String) = {
     // Append build.timestamp to Artifactory URL to get consistent build numbers (see https://github.com/sbt/sbt/issues/2088):
     val url2 = if(url.startsWith("file:")) url else url.replaceAll("/$", "") + ";build.timestamp=" + System.currentTimeMillis
-    Seq(publishTo in Global := Some("scala-pr-publish" at url2))
+
+    Seq(
+      publishTo in Global := Some("scala-pr-publish" at url2),
+      credentials in Global += Credentials("Artifactory Realm", "scala-ci.typesafe.com", "scala-ci", env("PRIVATE_REPO_PASS"))
+    )
   }
 
   /** Like `Def.sequential` but accumulate all results */

diff --git a/scripts/common b/scripts/common
@@ -159,19 +159,36 @@ EOF
 # Takes a variable number of additional repositories as argument.
 # See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html
 function generateRepositoriesConfig() {
-  jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"}
   sbtRepositoryConfig="$scriptsDir/sbt-repositories-config"
   echo > "$sbtRepositoryConfig" '[repositories]'
   if [[ $# -gt 0 ]]; then
     for i in $(seq 1 $#); do
       echo >> "$sbtRepositoryConfig" "  script-repo-$i: ${!i}"
     done
   fi
+
+  if [ "${TRAVIS}" != "true" ]; then
+    jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"}
+    echo "jcenter-cache: $jcenterCacheUrl" >> "$sbtRepositoryConfig"
+  fi
+
   cat >> "$sbtRepositoryConfig" << EOF
-  jcenter-cache: $jcenterCacheUrl
-  typesafe-ivy-releases: https://repo.lightbend.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
-  sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
-  maven-central
   local
+  maven-central
+  typesafe-ivy-releases-boot: https://repo.lightbend.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
+  typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
+  sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
 EOF
 }
+
+
+# https://github.com/travis-ci/docs-travis-ci-com/issues/949
+travis_fold_start() {
+  echo ""
+  echo -e "travis_fold:start:$1\033[33;1m$2\033[0m"
+}
+
+travis_fold_end() {
+  echo -e "\ntravis_fold:end:$1\r"
+  echo ""
+}
diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap
@@ -77,13 +77,14 @@ publishSonatypeTaskCore=${publishSonatypeTaskCore-"publishSigned"}
 publishSonatypeTaskModules=${publishSonatypeTaskModules-"publishSigned"}
 
 forceRebuild=${forceRebuild-no}
-
 sbtBuildTask=${sbtBuildTask-"testAll"} # TESTING leave empty to avoid the sanity check
 testStability=${testStability-yes}
 
 clean="clean" # TESTING leave empty to speed up testing
 
-baseDir=${WORKSPACE-`pwd`}
+WORKSPACE=${WORKSPACE-`pwd`}
+baseDir=${WORKSPACE}
+
 scriptsDir="$baseDir/scripts"
 . $scriptsDir/common
 
@@ -99,7 +100,9 @@ mkdir -p $baseDir/resolutionScratch_
 # repo to publish builds
 integrationRepoUrl=${integrationRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-integration/"}
 
-generateRepositoriesConfig $integrationRepoUrl
+if [ "${TRAVIS}" != "true" ]; then
+  generateRepositoriesConfig $integrationRepoUrl
+fi
 
 # ARGH trying to get this to work on multiple versions of sbt-extras...
 # the old version (on jenkins, and I don't want to upgrade for risk of breaking other builds) honors -sbt-dir
@@ -152,19 +155,22 @@ function st_stagingRepoClose() {
 #### sbt tools
 
 sbtBuild() {
-  echo "### sbtBuild: "$SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@"
-  $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1
+  travis_fold_start build "Building $(basename $PWD) with $@"
+  $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@"
+  travis_fold_end build
 }
 
 sbtResolve() {
   cd $baseDir/resolutionScratch_
   touch build.sbt
   # Can be set to `full` if a module requires cross-versioning against the full Scala version, like the continuations plugin used to.
   cross=${4-binary}
-  echo "### sbtResolve: $SBT_CMD -no-colors $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross"
+  # echo "### sbtResolve: $SBT_CMD -no-colors $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross"
+  travis_fold_start resolve "Resolving \"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross"
   $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" \
     "set libraryDependencies := Seq(\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross)" \
-      'show update' >> $baseDir/logs/resolution 2>&1
+      'show update'
+  travis_fold_end resolve
 }
 
 # Oh boy... can't use scaladoc to document scala-xml if scaladoc depends on the same version of scala-xml.
@@ -229,15 +235,15 @@ buildScalaCheck(){
 
 # build modules, using ${buildTasks[@]} (except for ScalaCheck, which is hard-coded to publish to artifactory)
 buildModules() {
-  publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"publish-repo\" at \"$integrationRepoUrl\")")
+  publishTasks=('set credentials += Credentials("Artifactory Realm", "scala-ci.typesafe.com", "scala-ci", System.getenv("PRIVATE_REPO_PASS"))' "set every publishTo := Some(\"publish-repo\" at \"$integrationRepoUrl\")")
   buildTasks=($publishPrivateTask)
   buildXML
   # buildScalaCheck
   buildPartest
 }
 
 buildPublishedModules() {
-  publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-sonatype")' "set pgpPassphrase := Some(Array.empty)")
+  publishTasks=('set credentials += Credentials("Sonatype Nexus Repository Manager", "oss.sonatype.org", System.getenv("SONA_USER"), System.getenv("SONA_PASS"))' "set pgpPassphrase := Some(Array.empty)")
   buildTasks=($publishSonatypeTaskModules)
   buildXML
   buildPartest
@@ -283,7 +289,9 @@ determineScalaVersion() {
   if [ -z "$SCALA_VER_BASE" ]; then
     echo "No SCALA_VER_BASE specified."
 
+    travis_fold_start determineScalaVersion "Determining Scala version"
     $SBT_CMD $sbtArgs 'set baseVersionSuffix in Global := "SHA"' generateBuildCharacterPropertiesFile
+    travis_fold_end determineScalaVersion
     parseScalaProperties "buildcharacter.properties"
     SCALA_VER_BASE="$maven_version_base"
     SCALA_VER_SUFFIX="$maven_version_suffix"
@@ -376,8 +384,6 @@ bootstrap() {
 
   #### (Optional) STARR.
   if [ ! -z "$STARR_REF" ]; then
-      echo "### Building STARR"
-
       STARR_DIR=./scala-starr
       STARR_VER_SUFFIX="-$(git rev-parse --short $STARR_REF)-starr"
       STARR_VER=$SCALA_VER_BASE$STARR_VER_SUFFIX
@@ -386,21 +392,24 @@ bootstrap() {
         git clone --reference $WORKSPACE/.git $WORKSPACE/.git $STARR_DIR
         cd $STARR_DIR
         git co $STARR_REF
-        $SBT_CMD -no-colors $sbtArgs --warn "setupBootstrapStarr $integrationRepoUrl $STARR_VER" $clean publish >> $baseDir/logs/builds 2>&1
+        travis_fold_start starr "Building starr"
+        $SBT_CMD -no-colors $sbtArgs --warn "setupBootstrapStarr $integrationRepoUrl $STARR_VER" $clean publish
+        travis_fold_end starr
       )
   fi
 
   #### LOCKER
 
-  echo "### Building locker"
-
   # for bootstrapping, publish core (or at least smallest subset we can get away with)
   # so that we can build modules with this version of Scala and publish them locally
   # must publish under $SCALA_VER so that the modules will depend on this (binary) version of Scala
   # publish more than just core: partest needs scalap
   # in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler
   if [ ! -z "$STARR_VER" ]; then SET_STARR=-Dstarr.version=$STARR_VER; fi
-  $SBT_CMD -no-colors $sbtArgs $SET_STARR --warn "setupBootstrapLocker $integrationRepoUrl $SCALA_VER" $clean publish >> $baseDir/logs/builds 2>&1
+
+  travis_fold_start locker "Building locker"
+  $SBT_CMD -no-colors $sbtArgs $SET_STARR --warn "setupBootstrapLocker $integrationRepoUrl $SCALA_VER" $clean publish
+  travis_fold_end locker
 
   echo "### Building modules using locker"
 
@@ -425,6 +434,7 @@ bootstrap() {
   cd $baseDir
   rm -rf build/
 
+  travis_fold_start quick "Building bootstrapped"
   $SBT_CMD $sbtArgs \
       --warn \
       -Dstarr.version=$SCALA_VER \
@@ -434,6 +444,7 @@ bootstrap() {
       $sbtBuildTask \
       dist/mkQuick \
       publish
+  travis_fold_end quick
 
   # clear ivy cache (and to be sure, local as well), so the next round of sbt builds sees the fresh scala
   rm -rf $baseDir/ivy2
@@ -443,8 +454,7 @@ bootstrap() {
 }
 
 testStability() {
-  echo "### Testing stability"
-
+  travis_fold_start stab "Testing stability"
   cd $baseDir
 
   # Run stability tests using the just built version as "quick" and a new version as "strap"
@@ -460,6 +470,8 @@ testStability() {
   mv build/quick build/strap
   mv quick1 build/quick
   $scriptsDir/stability-test.sh
+
+  travis_fold_end stab
 }
 
 # assumes we just bootstrapped, and current directory is $baseDir
@@ -469,15 +481,16 @@ testStability() {
 publishSonatype() {
   # stage to sonatype, along with all modules -Dmaven.version.suffix/-Dbuild.release not necessary,
   # since we're just publishing an existing build
-  echo "### Publishing core to sonatype"
+  travis_fold_start sona "Publishing core to sonatype"
   $SBT_CMD $sbtArgs \
       --warn \
       -Dstarr.version=$SCALA_VER \
       ${updatedModuleVersions[@]} \
       "setupBootstrapPublish $integrationRepoUrl $SCALA_VER" \
       $publishSonatypeTaskCore
+  travis_fold_end sona
 
-  echo "### Publishing modules to sonatype"
+  # echo "### Publishing modules to sonatype"
   # build/test/publish scala core modules to sonatype (this will start a new staging repo)
   # (was hoping we could make everything go to the same staging repo, but it's not timing that causes two staging repos to be opened)
   # NOTE: only publish those for which versions are set
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0")