diff --git a/.travis.yml b/.travis.yml index 923ffaf44cdf..76aa5f7968ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,37 @@ -# opt-in to Travis's newer/faster container-based infrastructure -sudo: false +sudo: required # GCE VMs have better performance (will be upgrading to premium VMs soon) # this builds the spec using jekyll # based on http://www.paperplanes.de/2013/8/13/deploying-your-jekyll-blog-to-s3-with-travis-ci.html -language: ruby + +language: scala +jdk: openjdk8 + +# the spec is built with jekyll rvm: - 2.2 -script: bundle exec jekyll build -s spec/ -d build/spec + +cache: + directories: + - $HOME/.ivy2/cache + - $HOME/.sbt + +script: + - (cd admin && ./init.sh) + - scripts/jobs/integrate/bootstrap + - bundle exec jekyll build -s spec/ -d build/spec + install: bundle install # cat /dev/urandom | head -c 10000 | openssl sha1 > ./secret # openssl aes-256-cbc -pass "file:./secret" -in id_dsa_spec212_b4096 -out spec/id_dsa_travis.enc -a # travis encrypt "PRIV_KEY_SECRET=`cat ./secret`" env: - - secure: "TuJOUtALynPd+MV1AuMeIpVb8BUBHr7Ul7FS48XhS2PyuTRpEBkSWybYcNg3AXyzmWDAuOjUxbaNMQBvP8vvehTbIYls5H5wTGKvj0D0TNVaPIXjF8bA8KyNat9xGNzhnWm2/2BMaWpKBJWRF7Jb+zHhijMYCJEbkMtoiE5R/mY=" + global: + - secure: "TuJOUtALynPd+MV1AuMeIpVb8BUBHr7Ul7FS48XhS2PyuTRpEBkSWybYcNg3AXyzmWDAuOjUxbaNMQBvP8vvehTbIYls5H5wTGKvj0D0TNVaPIXjF8bA8KyNat9xGNzhnWm2/2BMaWpKBJWRF7Jb+zHhijMYCJEbkMtoiE5R/mY=" + - secure: "T1fxtvLTxioyXJYiC/zVYdNYsBOt+0Piw+xE04rB1pzeKahm9+G2mISdcAyqv6/vze9eIJt6jNHHpKX32/Z3Cs1/Ruha4m3k+jblj3S0SbxV6ht2ieJXLT5WoUPFRrU68KXI8wqUadXpjxeJJV53qF2FC4lhfMUsw1IwwMhdaE8=" # PRIVATE_REPO_PASS + - secure: "feE5A8mYNpkNQKVwCj3aXrwjVrJWh/4ENpRfFlr2HOD9ORk1GORD5Yq907WZd+dTkYK54Lh1gA+qHOCIDgJHbi9ZLU+kjzEjtYKF6lQy6Wb0LI8smTOnAA6IWVVYifiXw8d66MI2MKZb2jjGeIzy8Q00SZjLhEGjLyTeCIB88Ws=" # SONA_USER + - secure: "ek3As5q2tL8UBXcxSBbv4v5YgsoPD41SCzPOSu72kzfbngyxgQxrcziU5pIM+Lib9KaWex7hVVWNL38tMyDbu+0OpDv8bPjMujzlDx5I2pJUfuOJo7QRYsJE1nsXcY4cA72cCLfbRcLEkvtDAhcdLSaUOqlyQe5BY4X4fY5eoPA=" # SONA_PASS + - secure: "dbAvl6KEuLwZ0MVQPZihFsPzCdiLbX0EFk3so+hcfEbksrmLQ1tn4X5ZM7Wy1UDR8uN9lxngEwHch7a7lKqpugzmXMew9Wnikr9WBWbJT77Z+XJ/jHI6YuiCRpRo+nvxXGp9Ry80tSIgx5eju0J83IaJL41BWlBkvyAd7YAHORI=" # GPG_SUBKEY_SECRET # ^^^ set PRIV_KEY_SECRET to password used to encrypt spec/id_dsa_travis.enc @@ -23,3 +41,8 @@ after_success: # using S3 would be simpler, but we want to upload to scala-lang.org # after_success: bundle exec s3_website push --headless + +before_cache: + # Cleanup the cached directories to avoid unnecessary cache updates + - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete + - find $HOME/.sbt -name "*.lock" -print -delete diff --git a/admin/files/credentials-private-repo b/admin/files/credentials-private-repo new file mode 100644 index 000000000000..ea665bb6b3f3 --- /dev/null +++ b/admin/files/credentials-private-repo @@ -0,0 +1,4 @@ +realm=Artifactory Realm +host=scala-ci.typesafe.com +user=scala-ci +password=${PRIVATE_REPO_PASS} \ No newline at end of file diff --git a/admin/files/credentials-sonatype b/admin/files/credentials-sonatype new file mode 100644 index 000000000000..906466c4054d --- /dev/null +++ b/admin/files/credentials-sonatype @@ -0,0 +1,4 @@ +realm=Sonatype Nexus Repository Manager +host=oss.sonatype.org +user=${SONA_USER} +password=${SONA_PASS} diff --git a/admin/files/gpg.sbt b/admin/files/gpg.sbt new file mode 100644 index 000000000000..2efcc4b691e8 --- /dev/null +++ b/admin/files/gpg.sbt @@ -0,0 +1 @@ +addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0") diff --git a/admin/files/gpg_subkey.enc b/admin/files/gpg_subkey.enc new file mode 100644 index 000000000000..de7e4ea4f402 Binary files /dev/null and b/admin/files/gpg_subkey.enc differ diff --git a/admin/files/m2-settings.xml b/admin/files/m2-settings.xml new file mode 100644 index 000000000000..5c54c2d8188e --- /dev/null +++ b/admin/files/m2-settings.xml @@ -0,0 +1,31 @@ + + + + + sonatype-nexus + ${SONA_USER} + ${SONA_PASS} + + + private-repo + scala-ci + ${PRIVATE_REPO_PASS} + + + + + + + codehaus-snapshots-mirror + Maven Codehaus snapshot repository + file:///codehaus-does-not-exist-anymore + codehaus-snapshots + + + diff --git a/admin/files/sonatype-curl b/admin/files/sonatype-curl new file mode 100644 index 000000000000..47f5e8c4cdd0 --- /dev/null +++ b/admin/files/sonatype-curl @@ -0,0 +1 @@ +user = ${SONA_USER}:${SONA_PASS} \ No newline at end of file diff --git a/admin/init.sh b/admin/init.sh new file mode 100755 index 000000000000..06f2b182e3cb --- /dev/null +++ b/admin/init.sh @@ -0,0 +1,30 @@ +#!/bin/bash + + +sensitive() { + perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/credentials-private-repo > ~/.credentials-private-repo + perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/credentials-sonatype > ~/.credentials-sonatype + perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/sonatype-curl > ~/.sonatype-curl + # perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/m2-settings.xml > ~/.m2/settings.xml -- not needed anymore (used for ide integration?) + + openssl aes-256-cbc -d -pass "pass:$GPG_SUBKEY_SECRET" -in files/gpg_subkey.enc | gpg --import +} + +# directories needed by sensitive part +# mkdir -p ~/.m2 -- not needed anymore (used for ide integration?) +mkdir -p ~/.ssh + +# don't let anything escape from the sensitive part (e.g. leak environment var by echoing to log on failure) +sensitive >/dev/null 2>&1 + +# pgp signing doesn't work without public key?? +gpg --keyserver pgp.mit.edu --recv-keys 0xa9052b1b6d92e560 + +# just to verify +gpg --list-keys +gpg --list-secret-keys + +mkdir -p ~/.sbt/0.13/plugins +cp files/gpg.sbt ~/.sbt/0.13/plugins/ + +export SBT_CMD=$(which sbt) diff --git a/project/ScriptCommands.scala b/project/ScriptCommands.scala index d15edc3f6786..f8644e74cfd5 100644 --- a/project/ScriptCommands.scala +++ b/project/ScriptCommands.scala @@ -6,6 +6,8 @@ import BuildSettings.autoImport._ /** Custom commands for use by the Jenkins scripts. This keeps the surface area and call syntax small. */ object ScriptCommands { + def env(key: String) = Option(System.getenv(key)).getOrElse("") + def all = Seq( setupPublishCore, setupValidateTest, @@ -80,7 +82,7 @@ object ScriptCommands { baseVersionSuffix in Global := "SPLIT", resolvers in Global += "scala-pr" at url, publishTo in Global := Some("sonatype-releases" at "https://oss.sonatype.org/service/local/staging/deploy/maven2"), - credentials in Global += Credentials(Path.userHome / ".credentials-sonatype"), + credentials in Global += Credentials("Sonatype Nexus Repository Manager", "oss.sonatype.org", env("SONA_USER"), env("SONA_PASS")), pgpPassphrase in Global := Some(Array.empty) ) ++ enableOptimizer } @@ -114,7 +116,11 @@ object ScriptCommands { private[this] def publishTarget(url: String) = { // Append build.timestamp to Artifactory URL to get consistent build numbers (see https://github.com/sbt/sbt/issues/2088): val url2 = if(url.startsWith("file:")) url else url.replaceAll("/$", "") + ";build.timestamp=" + System.currentTimeMillis - Seq(publishTo in Global := Some("scala-pr-publish" at url2)) + + Seq( + publishTo in Global := Some("scala-pr-publish" at url2), + credentials in Global += Credentials("Artifactory Realm", "scala-ci.typesafe.com", "scala-ci", env("PRIVATE_REPO_PASS")) + ) } /** Like `Def.sequential` but accumulate all results */ diff --git a/scripts/common b/scripts/common index 316d8ed5a0fb..d8903a7d8b05 100644 --- a/scripts/common +++ b/scripts/common @@ -159,7 +159,6 @@ EOF # Takes a variable number of additional repositories as argument. # See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html function generateRepositoriesConfig() { - jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} sbtRepositoryConfig="$scriptsDir/sbt-repositories-config" echo > "$sbtRepositoryConfig" '[repositories]' if [[ $# -gt 0 ]]; then @@ -167,11 +166,29 @@ function generateRepositoriesConfig() { echo >> "$sbtRepositoryConfig" " script-repo-$i: ${!i}" done fi + + if [ "${TRAVIS}" != "true" ]; then + jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} + echo "jcenter-cache: $jcenterCacheUrl" >> "$sbtRepositoryConfig" + fi + cat >> "$sbtRepositoryConfig" << EOF - jcenter-cache: $jcenterCacheUrl - typesafe-ivy-releases: https://repo.lightbend.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly - sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] - maven-central local + maven-central + typesafe-ivy-releases-boot: https://repo.lightbend.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly + typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] + sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] EOF } + + +# https://github.com/travis-ci/docs-travis-ci-com/issues/949 +travis_fold_start() { + echo "" + echo -e "travis_fold:start:$1\033[33;1m$2\033[0m" +} + +travis_fold_end() { + echo -e "\ntravis_fold:end:$1\r" + echo "" +} diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap index fd7f720a9457..0f41dd3939b7 100755 --- a/scripts/jobs/integrate/bootstrap +++ b/scripts/jobs/integrate/bootstrap @@ -77,13 +77,14 @@ publishSonatypeTaskCore=${publishSonatypeTaskCore-"publishSigned"} publishSonatypeTaskModules=${publishSonatypeTaskModules-"publishSigned"} forceRebuild=${forceRebuild-no} - sbtBuildTask=${sbtBuildTask-"testAll"} # TESTING leave empty to avoid the sanity check testStability=${testStability-yes} clean="clean" # TESTING leave empty to speed up testing -baseDir=${WORKSPACE-`pwd`} +WORKSPACE=${WORKSPACE-`pwd`} +baseDir=${WORKSPACE} + scriptsDir="$baseDir/scripts" . $scriptsDir/common @@ -99,7 +100,9 @@ mkdir -p $baseDir/resolutionScratch_ # repo to publish builds integrationRepoUrl=${integrationRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-integration/"} -generateRepositoriesConfig $integrationRepoUrl +if [ "${TRAVIS}" != "true" ]; then + generateRepositoriesConfig $integrationRepoUrl +fi # ARGH trying to get this to work on multiple versions of sbt-extras... # the old version (on jenkins, and I don't want to upgrade for risk of breaking other builds) honors -sbt-dir @@ -152,8 +155,9 @@ function st_stagingRepoClose() { #### sbt tools sbtBuild() { - echo "### sbtBuild: "$SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" - $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1 + travis_fold_start build "Building $(basename $PWD) with $@" + $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" + travis_fold_end build } sbtResolve() { @@ -161,10 +165,12 @@ sbtResolve() { touch build.sbt # Can be set to `full` if a module requires cross-versioning against the full Scala version, like the continuations plugin used to. cross=${4-binary} - echo "### sbtResolve: $SBT_CMD -no-colors $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" + # echo "### sbtResolve: $SBT_CMD -no-colors $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" + travis_fold_start resolve "Resolving \"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" \ "set libraryDependencies := Seq(\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross)" \ - 'show update' >> $baseDir/logs/resolution 2>&1 + 'show update' + travis_fold_end resolve } # Oh boy... can't use scaladoc to document scala-xml if scaladoc depends on the same version of scala-xml. @@ -229,7 +235,7 @@ buildScalaCheck(){ # build modules, using ${buildTasks[@]} (except for ScalaCheck, which is hard-coded to publish to artifactory) buildModules() { - publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"publish-repo\" at \"$integrationRepoUrl\")") + publishTasks=('set credentials += Credentials("Artifactory Realm", "scala-ci.typesafe.com", "scala-ci", System.getenv("PRIVATE_REPO_PASS"))' "set every publishTo := Some(\"publish-repo\" at \"$integrationRepoUrl\")") buildTasks=($publishPrivateTask) buildXML # buildScalaCheck @@ -237,7 +243,7 @@ buildModules() { } buildPublishedModules() { - publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-sonatype")' "set pgpPassphrase := Some(Array.empty)") + publishTasks=('set credentials += Credentials("Sonatype Nexus Repository Manager", "oss.sonatype.org", System.getenv("SONA_USER"), System.getenv("SONA_PASS"))' "set pgpPassphrase := Some(Array.empty)") buildTasks=($publishSonatypeTaskModules) buildXML buildPartest @@ -283,7 +289,9 @@ determineScalaVersion() { if [ -z "$SCALA_VER_BASE" ]; then echo "No SCALA_VER_BASE specified." + travis_fold_start determineScalaVersion "Determining Scala version" $SBT_CMD $sbtArgs 'set baseVersionSuffix in Global := "SHA"' generateBuildCharacterPropertiesFile + travis_fold_end determineScalaVersion parseScalaProperties "buildcharacter.properties" SCALA_VER_BASE="$maven_version_base" SCALA_VER_SUFFIX="$maven_version_suffix" @@ -376,8 +384,6 @@ bootstrap() { #### (Optional) STARR. if [ ! -z "$STARR_REF" ]; then - echo "### Building STARR" - STARR_DIR=./scala-starr STARR_VER_SUFFIX="-$(git rev-parse --short $STARR_REF)-starr" STARR_VER=$SCALA_VER_BASE$STARR_VER_SUFFIX @@ -386,21 +392,24 @@ bootstrap() { git clone --reference $WORKSPACE/.git $WORKSPACE/.git $STARR_DIR cd $STARR_DIR git co $STARR_REF - $SBT_CMD -no-colors $sbtArgs --warn "setupBootstrapStarr $integrationRepoUrl $STARR_VER" $clean publish >> $baseDir/logs/builds 2>&1 + travis_fold_start starr "Building starr" + $SBT_CMD -no-colors $sbtArgs --warn "setupBootstrapStarr $integrationRepoUrl $STARR_VER" $clean publish + travis_fold_end starr ) fi #### LOCKER - echo "### Building locker" - # for bootstrapping, publish core (or at least smallest subset we can get away with) # so that we can build modules with this version of Scala and publish them locally # must publish under $SCALA_VER so that the modules will depend on this (binary) version of Scala # publish more than just core: partest needs scalap # in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler if [ ! -z "$STARR_VER" ]; then SET_STARR=-Dstarr.version=$STARR_VER; fi - $SBT_CMD -no-colors $sbtArgs $SET_STARR --warn "setupBootstrapLocker $integrationRepoUrl $SCALA_VER" $clean publish >> $baseDir/logs/builds 2>&1 + + travis_fold_start locker "Building locker" + $SBT_CMD -no-colors $sbtArgs $SET_STARR --warn "setupBootstrapLocker $integrationRepoUrl $SCALA_VER" $clean publish + travis_fold_end locker echo "### Building modules using locker" @@ -425,6 +434,7 @@ bootstrap() { cd $baseDir rm -rf build/ + travis_fold_start quick "Building bootstrapped" $SBT_CMD $sbtArgs \ --warn \ -Dstarr.version=$SCALA_VER \ @@ -434,6 +444,7 @@ bootstrap() { $sbtBuildTask \ dist/mkQuick \ publish + travis_fold_end quick # clear ivy cache (and to be sure, local as well), so the next round of sbt builds sees the fresh scala rm -rf $baseDir/ivy2 @@ -443,8 +454,7 @@ bootstrap() { } testStability() { - echo "### Testing stability" - + travis_fold_start stab "Testing stability" cd $baseDir # Run stability tests using the just built version as "quick" and a new version as "strap" @@ -460,6 +470,8 @@ testStability() { mv build/quick build/strap mv quick1 build/quick $scriptsDir/stability-test.sh + + travis_fold_end stab } # assumes we just bootstrapped, and current directory is $baseDir @@ -469,15 +481,16 @@ testStability() { publishSonatype() { # stage to sonatype, along with all modules -Dmaven.version.suffix/-Dbuild.release not necessary, # since we're just publishing an existing build - echo "### Publishing core to sonatype" + travis_fold_start sona "Publishing core to sonatype" $SBT_CMD $sbtArgs \ --warn \ -Dstarr.version=$SCALA_VER \ ${updatedModuleVersions[@]} \ "setupBootstrapPublish $integrationRepoUrl $SCALA_VER" \ $publishSonatypeTaskCore + travis_fold_end sona - echo "### Publishing modules to sonatype" + # echo "### Publishing modules to sonatype" # build/test/publish scala core modules to sonatype (this will start a new staging repo) # (was hoping we could make everything go to the same staging repo, but it's not timing that causes two staging repos to be opened) # NOTE: only publish those for which versions are set