From 41e376a265398cc1e218c471a47f66ce9cfbc268 Mon Sep 17 00:00:00 2001 From: Adriaan Moors Date: Fri, 2 Mar 2018 15:28:47 +0100 Subject: [PATCH] Switch to Travis CI for building releases Both nightly and full releases migrate to Travis. PR validation remains on our Jenkins cluster for now. Main difference is that we don't use our artifactory as a cache, nor do we need to wipe stuff, since Travis gives us a fresh machine. We do cache the ivy/sbt cache. TODO: decide whether to publish a "mergely" (a nightly for each merge, or skip this script on merge and use a scheduled job for publishing the nightly.) Note: we don't use `travis encrypt-file` because it nukes the iv/key variables on each invocation.. too much magic Instead, I did: ``` cat /dev/urandom | head -c 10000 | openssl sha1 > ./secret openssl aes-256-cbc -pass "file:./secret" -in gpg_subkey -out admin/files/gpg_subkey.enc travis encrypt "GPG_SUBKEY_SECRET=$(cat ./secret)" ``` --- .travis.yml | 33 ++++++++++++++--- admin/files/credentials-private-repo | 4 +++ admin/files/credentials-sonatype | 4 +++ admin/files/gpg.sbt | 1 + admin/files/gpg_subkey.enc | Bin 0 -> 7328 bytes admin/files/m2-settings.xml | 31 ++++++++++++++++ admin/files/sonatype-curl | 1 + admin/init.sh | 30 ++++++++++++++++ project/ScriptCommands.scala | 10 ++++-- scripts/common | 27 +++++++++++--- scripts/jobs/integrate/bootstrap | 51 +++++++++++++++++---------- 11 files changed, 161 insertions(+), 31 deletions(-) create mode 100644 admin/files/credentials-private-repo create mode 100644 admin/files/credentials-sonatype create mode 100644 admin/files/gpg.sbt create mode 100644 admin/files/gpg_subkey.enc create mode 100644 admin/files/m2-settings.xml create mode 100644 admin/files/sonatype-curl create mode 100755 admin/init.sh diff --git a/.travis.yml b/.travis.yml index 923ffaf44cdf..76aa5f7968ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,37 @@ -# opt-in to Travis's newer/faster container-based infrastructure -sudo: false +sudo: required # GCE VMs have better performance (will be upgrading to premium VMs soon) # this builds the spec using jekyll # based on http://www.paperplanes.de/2013/8/13/deploying-your-jekyll-blog-to-s3-with-travis-ci.html -language: ruby + +language: scala +jdk: openjdk8 + +# the spec is built with jekyll rvm: - 2.2 -script: bundle exec jekyll build -s spec/ -d build/spec + +cache: + directories: + - $HOME/.ivy2/cache + - $HOME/.sbt + +script: + - (cd admin && ./init.sh) + - scripts/jobs/integrate/bootstrap + - bundle exec jekyll build -s spec/ -d build/spec + install: bundle install # cat /dev/urandom | head -c 10000 | openssl sha1 > ./secret # openssl aes-256-cbc -pass "file:./secret" -in id_dsa_spec212_b4096 -out spec/id_dsa_travis.enc -a # travis encrypt "PRIV_KEY_SECRET=`cat ./secret`" env: - - secure: "TuJOUtALynPd+MV1AuMeIpVb8BUBHr7Ul7FS48XhS2PyuTRpEBkSWybYcNg3AXyzmWDAuOjUxbaNMQBvP8vvehTbIYls5H5wTGKvj0D0TNVaPIXjF8bA8KyNat9xGNzhnWm2/2BMaWpKBJWRF7Jb+zHhijMYCJEbkMtoiE5R/mY=" + global: + - secure: "TuJOUtALynPd+MV1AuMeIpVb8BUBHr7Ul7FS48XhS2PyuTRpEBkSWybYcNg3AXyzmWDAuOjUxbaNMQBvP8vvehTbIYls5H5wTGKvj0D0TNVaPIXjF8bA8KyNat9xGNzhnWm2/2BMaWpKBJWRF7Jb+zHhijMYCJEbkMtoiE5R/mY=" + - secure: "T1fxtvLTxioyXJYiC/zVYdNYsBOt+0Piw+xE04rB1pzeKahm9+G2mISdcAyqv6/vze9eIJt6jNHHpKX32/Z3Cs1/Ruha4m3k+jblj3S0SbxV6ht2ieJXLT5WoUPFRrU68KXI8wqUadXpjxeJJV53qF2FC4lhfMUsw1IwwMhdaE8=" # PRIVATE_REPO_PASS + - secure: "feE5A8mYNpkNQKVwCj3aXrwjVrJWh/4ENpRfFlr2HOD9ORk1GORD5Yq907WZd+dTkYK54Lh1gA+qHOCIDgJHbi9ZLU+kjzEjtYKF6lQy6Wb0LI8smTOnAA6IWVVYifiXw8d66MI2MKZb2jjGeIzy8Q00SZjLhEGjLyTeCIB88Ws=" # SONA_USER + - secure: "ek3As5q2tL8UBXcxSBbv4v5YgsoPD41SCzPOSu72kzfbngyxgQxrcziU5pIM+Lib9KaWex7hVVWNL38tMyDbu+0OpDv8bPjMujzlDx5I2pJUfuOJo7QRYsJE1nsXcY4cA72cCLfbRcLEkvtDAhcdLSaUOqlyQe5BY4X4fY5eoPA=" # SONA_PASS + - secure: "dbAvl6KEuLwZ0MVQPZihFsPzCdiLbX0EFk3so+hcfEbksrmLQ1tn4X5ZM7Wy1UDR8uN9lxngEwHch7a7lKqpugzmXMew9Wnikr9WBWbJT77Z+XJ/jHI6YuiCRpRo+nvxXGp9Ry80tSIgx5eju0J83IaJL41BWlBkvyAd7YAHORI=" # GPG_SUBKEY_SECRET # ^^^ set PRIV_KEY_SECRET to password used to encrypt spec/id_dsa_travis.enc @@ -23,3 +41,8 @@ after_success: # using S3 would be simpler, but we want to upload to scala-lang.org # after_success: bundle exec s3_website push --headless + +before_cache: + # Cleanup the cached directories to avoid unnecessary cache updates + - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete + - find $HOME/.sbt -name "*.lock" -print -delete diff --git a/admin/files/credentials-private-repo b/admin/files/credentials-private-repo new file mode 100644 index 000000000000..ea665bb6b3f3 --- /dev/null +++ b/admin/files/credentials-private-repo @@ -0,0 +1,4 @@ +realm=Artifactory Realm +host=scala-ci.typesafe.com +user=scala-ci +password=${PRIVATE_REPO_PASS} \ No newline at end of file diff --git a/admin/files/credentials-sonatype b/admin/files/credentials-sonatype new file mode 100644 index 000000000000..906466c4054d --- /dev/null +++ b/admin/files/credentials-sonatype @@ -0,0 +1,4 @@ +realm=Sonatype Nexus Repository Manager +host=oss.sonatype.org +user=${SONA_USER} +password=${SONA_PASS} diff --git a/admin/files/gpg.sbt b/admin/files/gpg.sbt new file mode 100644 index 000000000000..2efcc4b691e8 --- /dev/null +++ b/admin/files/gpg.sbt @@ -0,0 +1 @@ +addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0") diff --git a/admin/files/gpg_subkey.enc b/admin/files/gpg_subkey.enc new file mode 100644 index 0000000000000000000000000000000000000000..de7e4ea4f40246cea920785f9b7a3c869b02e317 GIT binary patch literal 7328 zcmV;R9AD#8VQh3|WM5ydc&=++@<4HWK1?dG?Gajsu`p>+6#-Nx60jQAj8Vr+x%A!& zPDGh+%<0?TMNX!xFIoAy&SeoWeymTk%>h(3H~fIx;urknsV*5F+lIDiuH1b>F~QKG zcVDtKQ!CG~)GC_yYIvb+U{c=@SH^){&sfV;!23oiRjTDU0lRYaTp8W|uPax3cXKJO zE24kg2<^T^}v44sJ#=kkW7-hEjo;6{LQ_E1on=;rS37z>H`-`3dH+m!=NqAtenJzH5~ zYE1yCd$_AVMVhQCqb~&k<@T9#`YmzLQ_-8h>Zyt#~C;~P^*3&9v?@qjRN09O_ z$u3!y?@WBVcL+5L@?}zU0tL;myy<|MXsy4WG6h^T&MK$*-#0iJ;G@etoL_a zE;q;03k|(UKoM74Aku-NNhb+WzIMWn?bNbq>Y}U{2X(eWI+~)aPY*JJJagQv*X}AN z{h-A@0k@}`0GL*++5fzS*tyo+j5a-lO2}O2QOw3sH=-smv2uex>?Z5pIdk%V#;WU? zJYQaeb%H?}B8Q6yjU5hTU|^dH7V6Q9HP1yn)^1Buv{Y%3K$59cZ@k#O-J33+=mz2U z&bxvR9Ltq;_9oIX=AIvjh)kdNJ%A@#A)(#;2qXRM-i-g3)4(b{HI0~jeI`Njgj8#L zWp3rHhr*9U1Z57(uU;?fRZHVN4cB>gDO{^H?D>AM4MaJH;IPa(2-}8O8cQvjTQL9p zne9vH%TAZ8pL``~zoDR8YDSj4IU&~Xo@CIovFBZFX*wS1UDcZ*guE-0?cjWFyHJSUim(G4TdR43|*e&@?J8-?Z=APp4g< zriutI`xkyg)O#23DNsKlNoy~sPqsZUVo~j_asuNI|DZ(j<<`GlF@=i+O@DY5@p>!o zp507$c6&p)KvR+QG>HMBD|ixKWqjPZ5OE;?Pl{JoOK)~lGQ}dgXG>Z#OfhjdeYH_4 zU`gEkpErfy7Yic<)VGw2=loPiIjQbciculrmxHbbR=7?ckc~bzj2}qhT8W$h9xvg! z*zhFwS>~}<>SVK+M1eaH>oyfT3#~B|-4CS_LPN5MR83?K8_BCKVkF6fxa0D;I!`VU zxc3q9=mgqRJX6T~PWv>h+7gEnyBv#P=u1 zv%IMD$57$%^5Jo;tflCK9!;Td#-kVuCdQdD@@LS48z5sz`tclIwtrs&@C`Qnb%rqv zHayO9cu142Z_!oh`H248r~HLR80eWVs)9duUR-p}mlb*o zt90&bgg&SA*$-OWfj>k-C_RTp&7TY zFG&83qU|hPk}_`s+XOLp&zZ`elBcc6<80%tO?H!$)&Zsb-KuP}JH*fDa3_x`@v{gn zy+{N;L@>dX+S%8rY1*@Nn1)@Vw~>l!!r`^F9k|0w?XY77+d3*i{Txf|hbvPAz5 zkPAP8@_4DeLbWdEiVn{LO~&siPKa#Kj$NbsHq(-DjS9R_;I^K__ z>D^I7_8sROsntvJK9g_%1dxLxVKejVZhTwYDF;m>XeoE`aCm>ljDCi_klyM&EOS34Nyglm!z`j06d{#*$995b?T-# z^9`X=A!FlUL*{;o!3k^iv%}(7sp;=Tro`|Avk_1pm{i+s`%Ht`isD4*y)NsZ!CHF# zam=`dOF*8xyTs6FD?DZ5yK^#ZGyV$B5%xb`Xv{HnZR-{Cx*d=r%N74hl4y8*j#H!O zWx_)i8ri?znP$_(>B$5Eul%B z-R(#)t=Y2xG~&TbmpT2~0uUeG#u1z$LddjVRrSyhyxmf6HRnK7?wI%gUM7kHe#L?I z7tOr!rLjj?jAvwsWw%^3QB9y5TolJhb5a z=H9xgOw)=IW#ihZXOjB|2I~3;H*FE@u35#xGPk|^ZW?mjha=vKgk=%WN2tr_1kob4 zn#l1F=SvuEddGS?*Xe1?G=DJMY^_PWmp8Piaom47suEx#GWhbfguTtPiRhy5a0zLE z=hBB0E~d^$n7EYaD|!V1e#c5(aq%$)L!fZ2lm!dhP7YE9#Zo2{LXpYyB`-XbLpcY+ z4)Y*3JkSim6y#M|uq=&aV=srLI)WxitnA>}G}@g)j>+)CO%&t-lRsy{zYmW$upB4Q z#%{a}@(iEz1f0Y$x{CBXjlV>1@Hn%W#KGMERy{3xv*lDWc32T&5}Re*0t3(ZRSLo= ziVIePK1VV)6|+TYe3b%F1ED$4`PHpcUGk@L-^tApBip`lI$~1qucTDTC3Pi$mic_x zd%g>MVkpJmx2dwy1mtAmWY@_|SN=&To1dYwB-MdEw27O8#9sQ~=Y)i_Fb+J>b$_9J z|CQ+=+OSa@b!)3~fC{D{F;;3KYX7KULQ!aIUbS9hnR@nAmVp~sIOH^;Ig{;#Ha(tu^ zP21kRW9CR!?&5McM&-T9C+MayEz}2!=?qQzNHSa0>nyVO@CH*BktTGn5V<&G zlKkk^28@8@hQ=8^J4nPr9AE2e`YPZZBywn<@4FL8W~2J3U}RNcByT=_gB9B6V6#DA zPhP_I1xo?*zy;51Sh#SBsip-_T`s4>Se5vSCEkGf_KN(dJRxIA)x(olA{yTm$kNVz_^t*?m+_xatlF|I?7~w#af}oMNfC*n34>*7T}s*d^6_s0mwGBB}y1&OUgRn zn`StlCZ`7*+J9}0jdxlU{CIK2B0aU&0_0we_a*X*GjLY9PD1?9l9Om)UuEucY&Q=W z?!h!kzmJGjkSMy#vkVz-r^Pg3BY`~?z3b?dyruiv?>ZbIicNX3mmx099a{evzkKsE zwxIS;vz(HAF_d*cPTX;jWt48J3St&9jsgb>iqb}4Nnkj9O<^1?Ybs+l8#k0t4a5Cl z_!Ne;9$cIiYen7VGe(rcJS~Y@i^9@Bv|5lB9Hs4W~_bpYq`NBVL70X9p~s>pF_A{lEmKrf9L9!4L?a2 zmN)-SRq1sJvJr-Yn~^X*&>{gW-*%%*=RlJ!j}VD<#)V!u|4-HKHQBw%0L^) zyKOyDNuFU87Q8mg28acSe%+#O(E*-Z@n54B2v{gvEd5bk`gh})7=d>913QZt;4XQb zX*f&fJ;>LAhspxgD2s0jQ<+t)*kF9=G1nGZ6O*uzMnWdz6}0D7KJ?hagbc;~GYoI2 zsYaOtsXjJXl-BTGZwRbyBXLIDC~6qPqjnRahWi?4aW2ZnpU+@^7R$p^!j%)jLIL59 zX$6Z9hBAJ%U;4H|WeDm>qZ<>e2Ll{%jS z0ber_E9joA?CKT7o@dy+`@MXWF|&cwu6Eb==W`y=>2^D~^F`Yl?F@B&GZDJlzXq(? zNAN_|FTCIB=T9WtY+o}$>+%2P^VvPFopH=NY{%A=k+$~{S(2xVfkN=6JBH`x zo*z+A&RyQEbd1;1QV?C1aiu;4NQi!WglfO6{y~f<90D%A=sT;X7%Ka6c)nd4C-k9( z&Y)s$6u++Ei*ncwo!Zh@R}=p^m^RnGH(hfOBp0H6|83{vjSvJJ)qS4?Z1PToFd(WFORPoW_`nPY`hN3s#pZhB#=v98~Q%FSrZ zY`#Y?6eb0$jsnLU!S+Xbhz8{v;g7C2`RAOOCITl-TIgf{+Z)pDAX*%<`~6fD-#qh( z!J0g}mIu=*Eb>>$16aGtrTmgCg`A%jZ^n#E4( zt;9VCbP}_5E5)u!S6DuxeKiWun3a#ta)L7ezbv&J84j=OpxzDiuJD5^9EZ0V{DjJt zxqj6rkH&92ZEVT#Ka5>Fcq?r`q4}HjZPYNbks)9zojhJkQre#{2bwEWbzB*xqIX1S zGj_dO0B_G%K;%&J7xdghGaLx@KpdIx+4}yE^DcU%nNW}J-WhSyN6Yqnh*qsyyt!W% zY3$f1x`v_H@{W#nhVMa7bFvkg5o-ABR(3*tL_wZjYpZYXfVH=HF30ePXu`Pk|9x3< zA&77FiFL&KT7-Ole`hQJJO&lba=LwYD&Hu$^cbUDK+v*(=uj-m(pbh`T!{E!n_~rG z-$O&uLuCN__XWaXE`Spkm3p;rQi?EdM>op7D zWkBn|`U-Z>V7Uxg|Nn5a=u&t<#Ir4BCQ1wUv1%a$4YbQJ<E%9dnqjQX0@TtW4mm2rt$y*?zls zmUPMf95mWm2DJ3N!(iBj_PtK7(8G_$V3;L8Nfj|zV7I6jumP4N<+mIJV+!PVY3g%<4`-@jbtc=qM`f_DbuL0KMvwi_Xh zDcyqUksDhv)dR6iAvHrH%npB(lIynZ5{VG(Np>J9Rh436~RgMNeWeVbw1(3}*#!|=bZolR|E zIAa>>eg`!5Si;OQ_o22#k47%^WvGMxN#+oq-Ri%hyYgW%?Nf-gofe5Q5t=I3t^SD&lT(Nu zExi#_n-XFoinTpnGgdpdR7tYlP93vhUZ;y|91OJpf50ql6BoD>>2GP?@7_u5A!Fg1 z#cO*vBJr;vJNd|M@Gt&WeP)d$jV#UC@!3yDib~%k7@KdyJ&orj#{PA1gc+R@GeCOf z&oKWk=acHg+(V<=T{mxz54h7-Z{1$hF#BORtW0kh(vi(Ho%Zf13>{UJ(IbT&X%*Pv z)7%I43=6#-=q4Dv;HI0BC`eY7^beX!XxOYTdhX|Oti;A-&7fQuX1x^4b6BS2;>teZ^HEEBi z{G@j6=KAc^M|I=xWnsvg*3Ag3`Oe!Mpv(hAe)Y`!+3{iIC~4WBX`PSJ_2B8oBoI@^ z!4YFv;HQf>>F3VIP+ff>Db6NKn(OJfrIDa)fg613z&E)Qc=^y3uj-#SK9`Sd}mf_T`zw_Q*c;e)DcID0ut#Qm){rFGvNVJc%&x?aXGY zJ-*^XN37Z1br--poD3JQn_6IJ7$STHpG0JqDp}V^wD(7mS1YI5_jYQwHp%lEjOa80 zWRue;4?fT9{&l;(I|4_&ly6w8EbqS5(Qx(G7 zhhc`(;|eV}(oIMs@b;5=DWCTP9dITOU#T}8vz>ii6| z#AgmjK1fi@dPvvEkGUp{a{gErn?E#R5+Rn(p|-Ub$J!T2bGW;wW8x3zB)45(fdh72 zgM{?a=6Xv2r0^B8TP~;UnV9UOV$#8wC$%Su&~`;!@Wv4l`_(iHSkd6W=M_yA8YhpV zRa~4%jv;G^Aj@@L?sZA+_k{Fzj8C#z74zm%(6lAWUlV*k5_8C(@~JL&F{@!d zPW4J0_K2GCHJVv3<4C;YyIorvs=LO&fo=l$MAxD0i|NkN*1gP`Wl5CM!h{|r&nHpR z*$r_t#Y`hk-Eb>t0`VzJ)(%2y@arZXQ(%aRLD_b*9!?PcKp + + + + sonatype-nexus + ${SONA_USER} + ${SONA_PASS} + + + private-repo + scala-ci + ${PRIVATE_REPO_PASS} + + + + + + + codehaus-snapshots-mirror + Maven Codehaus snapshot repository + file:///codehaus-does-not-exist-anymore + codehaus-snapshots + + + diff --git a/admin/files/sonatype-curl b/admin/files/sonatype-curl new file mode 100644 index 000000000000..47f5e8c4cdd0 --- /dev/null +++ b/admin/files/sonatype-curl @@ -0,0 +1 @@ +user = ${SONA_USER}:${SONA_PASS} \ No newline at end of file diff --git a/admin/init.sh b/admin/init.sh new file mode 100755 index 000000000000..06f2b182e3cb --- /dev/null +++ b/admin/init.sh @@ -0,0 +1,30 @@ +#!/bin/bash + + +sensitive() { + perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/credentials-private-repo > ~/.credentials-private-repo + perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/credentials-sonatype > ~/.credentials-sonatype + perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/sonatype-curl > ~/.sonatype-curl + # perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/m2-settings.xml > ~/.m2/settings.xml -- not needed anymore (used for ide integration?) + + openssl aes-256-cbc -d -pass "pass:$GPG_SUBKEY_SECRET" -in files/gpg_subkey.enc | gpg --import +} + +# directories needed by sensitive part +# mkdir -p ~/.m2 -- not needed anymore (used for ide integration?) +mkdir -p ~/.ssh + +# don't let anything escape from the sensitive part (e.g. leak environment var by echoing to log on failure) +sensitive >/dev/null 2>&1 + +# pgp signing doesn't work without public key?? +gpg --keyserver pgp.mit.edu --recv-keys 0xa9052b1b6d92e560 + +# just to verify +gpg --list-keys +gpg --list-secret-keys + +mkdir -p ~/.sbt/0.13/plugins +cp files/gpg.sbt ~/.sbt/0.13/plugins/ + +export SBT_CMD=$(which sbt) diff --git a/project/ScriptCommands.scala b/project/ScriptCommands.scala index d15edc3f6786..f8644e74cfd5 100644 --- a/project/ScriptCommands.scala +++ b/project/ScriptCommands.scala @@ -6,6 +6,8 @@ import BuildSettings.autoImport._ /** Custom commands for use by the Jenkins scripts. This keeps the surface area and call syntax small. */ object ScriptCommands { + def env(key: String) = Option(System.getenv(key)).getOrElse("") + def all = Seq( setupPublishCore, setupValidateTest, @@ -80,7 +82,7 @@ object ScriptCommands { baseVersionSuffix in Global := "SPLIT", resolvers in Global += "scala-pr" at url, publishTo in Global := Some("sonatype-releases" at "https://oss.sonatype.org/service/local/staging/deploy/maven2"), - credentials in Global += Credentials(Path.userHome / ".credentials-sonatype"), + credentials in Global += Credentials("Sonatype Nexus Repository Manager", "oss.sonatype.org", env("SONA_USER"), env("SONA_PASS")), pgpPassphrase in Global := Some(Array.empty) ) ++ enableOptimizer } @@ -114,7 +116,11 @@ object ScriptCommands { private[this] def publishTarget(url: String) = { // Append build.timestamp to Artifactory URL to get consistent build numbers (see https://github.com/sbt/sbt/issues/2088): val url2 = if(url.startsWith("file:")) url else url.replaceAll("/$", "") + ";build.timestamp=" + System.currentTimeMillis - Seq(publishTo in Global := Some("scala-pr-publish" at url2)) + + Seq( + publishTo in Global := Some("scala-pr-publish" at url2), + credentials in Global += Credentials("Artifactory Realm", "scala-ci.typesafe.com", "scala-ci", env("PRIVATE_REPO_PASS")) + ) } /** Like `Def.sequential` but accumulate all results */ diff --git a/scripts/common b/scripts/common index 316d8ed5a0fb..d8903a7d8b05 100644 --- a/scripts/common +++ b/scripts/common @@ -159,7 +159,6 @@ EOF # Takes a variable number of additional repositories as argument. # See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html function generateRepositoriesConfig() { - jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} sbtRepositoryConfig="$scriptsDir/sbt-repositories-config" echo > "$sbtRepositoryConfig" '[repositories]' if [[ $# -gt 0 ]]; then @@ -167,11 +166,29 @@ function generateRepositoriesConfig() { echo >> "$sbtRepositoryConfig" " script-repo-$i: ${!i}" done fi + + if [ "${TRAVIS}" != "true" ]; then + jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} + echo "jcenter-cache: $jcenterCacheUrl" >> "$sbtRepositoryConfig" + fi + cat >> "$sbtRepositoryConfig" << EOF - jcenter-cache: $jcenterCacheUrl - typesafe-ivy-releases: https://repo.lightbend.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly - sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] - maven-central local + maven-central + typesafe-ivy-releases-boot: https://repo.lightbend.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly + typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] + sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] EOF } + + +# https://github.com/travis-ci/docs-travis-ci-com/issues/949 +travis_fold_start() { + echo "" + echo -e "travis_fold:start:$1\033[33;1m$2\033[0m" +} + +travis_fold_end() { + echo -e "\ntravis_fold:end:$1\r" + echo "" +} diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap index fd7f720a9457..0f41dd3939b7 100755 --- a/scripts/jobs/integrate/bootstrap +++ b/scripts/jobs/integrate/bootstrap @@ -77,13 +77,14 @@ publishSonatypeTaskCore=${publishSonatypeTaskCore-"publishSigned"} publishSonatypeTaskModules=${publishSonatypeTaskModules-"publishSigned"} forceRebuild=${forceRebuild-no} - sbtBuildTask=${sbtBuildTask-"testAll"} # TESTING leave empty to avoid the sanity check testStability=${testStability-yes} clean="clean" # TESTING leave empty to speed up testing -baseDir=${WORKSPACE-`pwd`} +WORKSPACE=${WORKSPACE-`pwd`} +baseDir=${WORKSPACE} + scriptsDir="$baseDir/scripts" . $scriptsDir/common @@ -99,7 +100,9 @@ mkdir -p $baseDir/resolutionScratch_ # repo to publish builds integrationRepoUrl=${integrationRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-integration/"} -generateRepositoriesConfig $integrationRepoUrl +if [ "${TRAVIS}" != "true" ]; then + generateRepositoriesConfig $integrationRepoUrl +fi # ARGH trying to get this to work on multiple versions of sbt-extras... # the old version (on jenkins, and I don't want to upgrade for risk of breaking other builds) honors -sbt-dir @@ -152,8 +155,9 @@ function st_stagingRepoClose() { #### sbt tools sbtBuild() { - echo "### sbtBuild: "$SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" - $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1 + travis_fold_start build "Building $(basename $PWD) with $@" + $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" + travis_fold_end build } sbtResolve() { @@ -161,10 +165,12 @@ sbtResolve() { touch build.sbt # Can be set to `full` if a module requires cross-versioning against the full Scala version, like the continuations plugin used to. cross=${4-binary} - echo "### sbtResolve: $SBT_CMD -no-colors $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" + # echo "### sbtResolve: $SBT_CMD -no-colors $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" + travis_fold_start resolve "Resolving \"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" \ "set libraryDependencies := Seq(\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross)" \ - 'show update' >> $baseDir/logs/resolution 2>&1 + 'show update' + travis_fold_end resolve } # Oh boy... can't use scaladoc to document scala-xml if scaladoc depends on the same version of scala-xml. @@ -229,7 +235,7 @@ buildScalaCheck(){ # build modules, using ${buildTasks[@]} (except for ScalaCheck, which is hard-coded to publish to artifactory) buildModules() { - publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"publish-repo\" at \"$integrationRepoUrl\")") + publishTasks=('set credentials += Credentials("Artifactory Realm", "scala-ci.typesafe.com", "scala-ci", System.getenv("PRIVATE_REPO_PASS"))' "set every publishTo := Some(\"publish-repo\" at \"$integrationRepoUrl\")") buildTasks=($publishPrivateTask) buildXML # buildScalaCheck @@ -237,7 +243,7 @@ buildModules() { } buildPublishedModules() { - publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-sonatype")' "set pgpPassphrase := Some(Array.empty)") + publishTasks=('set credentials += Credentials("Sonatype Nexus Repository Manager", "oss.sonatype.org", System.getenv("SONA_USER"), System.getenv("SONA_PASS"))' "set pgpPassphrase := Some(Array.empty)") buildTasks=($publishSonatypeTaskModules) buildXML buildPartest @@ -283,7 +289,9 @@ determineScalaVersion() { if [ -z "$SCALA_VER_BASE" ]; then echo "No SCALA_VER_BASE specified." + travis_fold_start determineScalaVersion "Determining Scala version" $SBT_CMD $sbtArgs 'set baseVersionSuffix in Global := "SHA"' generateBuildCharacterPropertiesFile + travis_fold_end determineScalaVersion parseScalaProperties "buildcharacter.properties" SCALA_VER_BASE="$maven_version_base" SCALA_VER_SUFFIX="$maven_version_suffix" @@ -376,8 +384,6 @@ bootstrap() { #### (Optional) STARR. if [ ! -z "$STARR_REF" ]; then - echo "### Building STARR" - STARR_DIR=./scala-starr STARR_VER_SUFFIX="-$(git rev-parse --short $STARR_REF)-starr" STARR_VER=$SCALA_VER_BASE$STARR_VER_SUFFIX @@ -386,21 +392,24 @@ bootstrap() { git clone --reference $WORKSPACE/.git $WORKSPACE/.git $STARR_DIR cd $STARR_DIR git co $STARR_REF - $SBT_CMD -no-colors $sbtArgs --warn "setupBootstrapStarr $integrationRepoUrl $STARR_VER" $clean publish >> $baseDir/logs/builds 2>&1 + travis_fold_start starr "Building starr" + $SBT_CMD -no-colors $sbtArgs --warn "setupBootstrapStarr $integrationRepoUrl $STARR_VER" $clean publish + travis_fold_end starr ) fi #### LOCKER - echo "### Building locker" - # for bootstrapping, publish core (or at least smallest subset we can get away with) # so that we can build modules with this version of Scala and publish them locally # must publish under $SCALA_VER so that the modules will depend on this (binary) version of Scala # publish more than just core: partest needs scalap # in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler if [ ! -z "$STARR_VER" ]; then SET_STARR=-Dstarr.version=$STARR_VER; fi - $SBT_CMD -no-colors $sbtArgs $SET_STARR --warn "setupBootstrapLocker $integrationRepoUrl $SCALA_VER" $clean publish >> $baseDir/logs/builds 2>&1 + + travis_fold_start locker "Building locker" + $SBT_CMD -no-colors $sbtArgs $SET_STARR --warn "setupBootstrapLocker $integrationRepoUrl $SCALA_VER" $clean publish + travis_fold_end locker echo "### Building modules using locker" @@ -425,6 +434,7 @@ bootstrap() { cd $baseDir rm -rf build/ + travis_fold_start quick "Building bootstrapped" $SBT_CMD $sbtArgs \ --warn \ -Dstarr.version=$SCALA_VER \ @@ -434,6 +444,7 @@ bootstrap() { $sbtBuildTask \ dist/mkQuick \ publish + travis_fold_end quick # clear ivy cache (and to be sure, local as well), so the next round of sbt builds sees the fresh scala rm -rf $baseDir/ivy2 @@ -443,8 +454,7 @@ bootstrap() { } testStability() { - echo "### Testing stability" - + travis_fold_start stab "Testing stability" cd $baseDir # Run stability tests using the just built version as "quick" and a new version as "strap" @@ -460,6 +470,8 @@ testStability() { mv build/quick build/strap mv quick1 build/quick $scriptsDir/stability-test.sh + + travis_fold_end stab } # assumes we just bootstrapped, and current directory is $baseDir @@ -469,15 +481,16 @@ testStability() { publishSonatype() { # stage to sonatype, along with all modules -Dmaven.version.suffix/-Dbuild.release not necessary, # since we're just publishing an existing build - echo "### Publishing core to sonatype" + travis_fold_start sona "Publishing core to sonatype" $SBT_CMD $sbtArgs \ --warn \ -Dstarr.version=$SCALA_VER \ ${updatedModuleVersions[@]} \ "setupBootstrapPublish $integrationRepoUrl $SCALA_VER" \ $publishSonatypeTaskCore + travis_fold_end sona - echo "### Publishing modules to sonatype" + # echo "### Publishing modules to sonatype" # build/test/publish scala core modules to sonatype (this will start a new staging repo) # (was hoping we could make everything go to the same staging repo, but it's not timing that causes two staging repos to be opened) # NOTE: only publish those for which versions are set