Rewrite Travis CI script.

Lots of changes:

    - When possible, we use the container infrastructure (sudo: false)
      rather than Google Compute Engine infrastructure (sudo: required).
      Unfortunately, we can't use GCE for the Linux builds, where
      reduced RAM available hoses are GHC build.

    - Switched from using ./Setup and old-style cabal to new-build.
      There are numerous great benefits but the best is that
      .cabal/store can be cached on Travis, leading to huge speedups
      on the build.  Downside is we need to string-and-ceiling-wax
      support for test/haddock/etc.

    - I stopped bootstrapping on every build we do; instead there
      is a separate bootstrap build we do to make sure that that
      is working.  This also speeds up the basic builds since
      we are not building Cabal/cabal-install multiple times.

    - There are some hacks.  The big one is setting CABAL_BUILDDIR
      explicitly; this smooths over quite a few infelicities in
      the current new-build implementation.

Signed-off-by: Edward Z. Yang <[email protected]>

.travis.yml

@@ -2,50 +2,72 @@
 # We specify language: c, so it doesn't default to e.g. ruby
 language: c
 
-sudo: required
+sudo: false
 
 # The following enables several GHC versions to be tested; often it's enough to
 # test only against the last release in a major GHC version. Feel free to omit
 # lines listings versions you don't need/want testing for.
 matrix:
   include:
-   - env: GHCVER=7.4.2
+   - env: GHCVER=none SCRIPT=meta
      os: linux
-   - env: GHCVER=7.6.3
+   # These don't have -dyn/-prof whitelisted yet, so we have to
+   # do the old-style installation
+   - env: GHCVER=7.4.2 SCRIPT=script
      os: linux
-   - env: GHCVER=7.8.4
+     sudo: required
+   - env: GHCVER=7.6.3 SCRIPT=script
      os: linux
-   - env: GHCVER=7.10.3
+     sudo: required
+   - env: GHCVER=7.8.4 SCRIPT=script
      os: linux
-   - env: GHCVER=8.0.1 TEST_OLDER=NO DEPLOY_DOCS=YES
+     sudo: required
+   # Ugh, we'd like to drop 'sudo: required' and use the
+   # apt plugin for the next two
+   # but the GCE instance we get has more memory, which makes
+   # a big difference
+   - env: GHCVER=7.10.3 SCRIPT=script
+     os: linux
+     sudo: required
+   - env: GHCVER=8.0.1 SCRIPT=script DEPLOY_DOCS=YES
+     sudo: required
+     os: linux
+   - env: GHCVER=8.0.1 SCRIPT=bootstrap
+     sudo: required
      os: linux
    # It's not worth the trouble to make older GHC work with clang's cpp
-   # Obviously TEST_OLDER doesn't work with OSX
+   # TEST_OLDER is not implemented on OSX
    #
    # Also we might want to specify OSX version
    # https://docs.travis-ci.com/user/osx-ci-environment/#OS-X-Version
-   - env: GHCVER=7.8.4
+   - env: GHCVER=7.8.4 SCRIPT=script
      os: osx
-   - env: GHCVER=7.10.3
+   - env: GHCVER=7.10.3 SCRIPT=script
      os: osx
-   - env: GHCVER=8.0.1
+   - env: GHCVER=8.0.1 SCRIPT=script
      os: osx
   allow_failures:
    - env: GHCVER=head
+     sudo: required
 
  # TODO add PARSEC_BUNDLED=YES when it's so
  # It seems pointless to run head if we're going to ignore the results.
  #- GHCVER=head
 
 # Note: the distinction between `before_install` and `install` is not important.
 before_install:
- - ./travis-install.sh
  - export PATH=/opt/ghc/$GHCVER/bin:$PATH
- - export PATH=$HOME/.ghc-install/$GHCVER/bin:$PATH;
+ - export PATH=$HOME/.ghc-install/$GHCVER/bin:$PATH
+ - export PATH=$HOME/bin:$PATH
+ - export PATH=$HOME/.cabal/bin:$PATH
+ - export PATH=/opt/cabal/1.24/bin:$PATH
+ - export PATH=/opt/happy/1.19.5/bin:$PATH
+ - ./travis-install.sh
 
  # Set up deployment to the haskell/cabal-website repo.
  # NB: these commands MUST be in .travis.yml, otherwise the secret key can be
  # leaked! See https://github.com/travis-ci/travis.rb/issues/423.
+ # umask to get the permissions to be 400.
  - if [ "x$TRAVIS_PULL_REQUEST" = "xfalse" -a "x$TRAVIS_BRANCH" = "xmaster" -a "x$DEPLOY_DOCS" = "xYES"  ]; then (umask 377 && openssl aes-256-cbc -K $encrypted_edaf6551664d_key -iv $encrypted_edaf6551664d_iv -in id_rsa_cabal_website.aes256.enc -out ~/.ssh/id_rsa -d); fi
 
 install:
@@ -57,7 +79,20 @@ install:
 # ./dist/setup/setup here instead of cabal-install to avoid breakage when the
 # build config format changed.
 script:
- - ./travis-script.sh -j
+ - ./travis-${SCRIPT}.sh
+
+cache:
+ directories:
+ - $HOME/.cabal/packages
+ - $HOME/.cabal/store
+ - $HOME/.cabal/bin
+
+# We remove the index because it churns quite a bit and we don't want
+# to pay the cost of repeatedly caching it even though we don't care
+# about most changing packages.
+before_cache:
+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/build-reports.log
+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/00-index*
 
 # Deploy Haddocks to the haskell/cabal-website repo.
 after_success:

Cabal/Distribution/Compat/Environment.hs

@@ -51,7 +51,7 @@ lookupEnv name = (Just `fmap` System.getEnv name) `catchIO` const (return Nothin
 -- empty string or contains an equals sign.
 setEnv :: String -> String -> IO ()
 setEnv key value_
-  | null value = error "Distribuiton.Compat.setEnv: empty string"
+  | null value = error "Distribution.Compat.setEnv: empty string"
   | otherwise  = setEnv_ key value
   where
     -- NOTE: Anything that follows NUL is ignored on both POSIX and Windows. We

cabal-install/tests/IntegrationTests.hs

@@ -9,7 +9,7 @@ module Main
 
 -- Modules from Cabal.
 import Distribution.Compat.CreatePipe (createPipe)
-import Distribution.Compat.Environment (setEnv)
+import Distribution.Compat.Environment (setEnv, getEnvironment)
 import Distribution.Compat.Internal.TempFile (createTempDirectory)
 import Distribution.Simple.Configure (findDistPrefOrDefault)
 import Distribution.Simple.Program.Builtin (ghcPkgProgram)
@@ -100,7 +100,11 @@ run cwd path args = do
     (hReadStdOut, hWriteStdOut) <- createPipe
     (hReadStdErr, hWriteStdErr) <- createPipe
     -- Run the process
-    pid <- runProcess path' args (Just cwd) Nothing Nothing (Just hWriteStdOut) (Just hWriteStdErr)
+    env0 <- getEnvironment
+    -- CABAL_BUILDDIR can interfere with test running, so
+    -- be sure to clear it out.
+    let env = filter ((/= "CABAL_BUILDDIR") . fst) env0
+    pid <- runProcess path' args (Just cwd) (Just env) Nothing (Just hWriteStdOut) (Just hWriteStdErr)
     -- Return the pid and read ends of the pipes
     return (pid, hReadStdOut, hReadStdErr)
   -- Read subprocess output using asynchronous threads; we need to

cabal.project.travis

@@ -0,0 +1,13 @@
+-- Force error messages to be better
+-- Parallel new-build error messages are non-existent.
+-- Turn off parallelization to get good errors.
+jobs: 1
+
+-- The -fno-warn-orphans is a hack to make Cabal-1.24
+-- build properly (unfortunately the flags here get applied
+-- to the dependencies too!)
+package Cabal
+  ghc-options: -Werror -fno-warn-orphans
+
+package cabal-install
+  ghc-options: -Werror

travis-bootstrap.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. ./travis-common.sh
+
+# ---------------------------------------------------------------------
+# Bootstrap cabal, to verify bootstrap.sh script works.
+# ---------------------------------------------------------------------
+
+bootstrap_jobs="-j"
+
+(cd cabal-install && timed env EXTRA_CONFIGURE_OPTS="" ./bootstrap.sh $bootstrap_jobs --no-doc)
+timed $HOME/.cabal/bin/cabal --version
+PATH=$HOME/.cabal/bin:$PATH
+
+# ---------------------------------------------------------------------
+# Verify that installation from tarball works.
+# ---------------------------------------------------------------------
+
+# The following scriptlet checks that the resulting source distribution can be
+# built & installed.
+install_from_tarball() {
+   SRC_TGZ=$(cabal info . | awk '{print $2 ".tar.gz";exit}') ;
+   export SRC_TGZ
+   if [ -f "dist/$SRC_TGZ" ]; then
+      cabal install --force-reinstalls $jobs "dist/$SRC_TGZ" -v2;
+   else
+      echo "expected 'dist/$SRC_TGZ' not found";
+      exit 1;
+   fi
+}
+
+timed cabal update
+
+echo Cabal
+(cd Cabal && timed cabal sdist)
+(cd Cabal && timed install_from_tarball)
+
+echo cabal-install
+(cd cabal-install && timed cabal sdist)
+(cd cabal-install && timed install_from_tarball)

travis-common.sh

@@ -0,0 +1,17 @@
+set -e
+
+CABAL_VERSION="1.25.0.0"
+
+# ---------------------------------------------------------------------
+# Timing / diagnostic output
+# ---------------------------------------------------------------------
+
+timed() {
+    echo "\$ $*"
+    start_time=$(date +%s)
+    $*
+    end_time=$(date +%s)
+    duration=$((end_time - start_time))
+    echo "$* took $duration seconds."
+    echo "----"
+}

travis-install.sh

@@ -5,10 +5,14 @@ travis_retry () {
     $*  || (sleep 1 && $*) || (sleep 2 && $*)
 }
 
+if [ "$GHCVER" = "none" ]; then
+    exit 0
+fi
+
 if [ "$TRAVIS_OS_NAME" = "linux" ]; then
     travis_retry sudo add-apt-repository -y ppa:hvr/ghc
     travis_retry sudo apt-get update
-    travis_retry sudo apt-get install --force-yes ghc-$GHCVER-prof ghc-$GHCVER-dyn
+    travis_retry sudo apt-get install --force-yes cabal-install-1.24 happy-1.19.5 ghc-$GHCVER-prof ghc-$GHCVER-dyn
     if [ "$TEST_OLDER" == "YES" ]; then travis_retry sudo apt-get install --force-yes ghc-7.0.4-prof ghc-7.0.4-dyn ghc-7.2.2-prof ghc-7.2.2-dyn; fi
 
 elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
@@ -50,6 +54,12 @@ elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
     make install;
     cd ..;
 
+    travis_retry curl -L https://www.haskell.org/cabal/release/cabal-install-1.24.0.0/cabal-install-1.24.0.0-x86_64-apple-darwin-yosemite.tar.gz -o cabal-install.tar.gz
+    TAR=$PWD/cabal-install.tar.gz
+    mkdir "${HOME}/bin"
+    (cd "${HOME}/bin" && tar -xzf "$TAR")
+    "${HOME}/bin/cabal" --version
+
 else
     echo "Not linux or osx: $TRAVIS_OS_NAME"
     false

travis-meta.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+. ./travis-common.sh
+
+# ---------------------------------------------------------------------
+# Check that auto-generated files/fields are up to date.
+# ---------------------------------------------------------------------
+
+# Regenerate the CONTRIBUTORS file.
+# Currently doesn't work because Travis uses --depth=50 when cloning.
+#./Cabal/misc/gen-authors.sh > AUTHORS
+
+# Regenerate the 'extra-source-files' field in Cabal.cabal.
+(cd Cabal && timed ./misc/gen-extra-source-files.sh Cabal.cabal)
+
+# Regenerate the 'extra-source-files' field in cabal-install.cabal.
+(cd cabal-install && ../Cabal/misc/gen-extra-source-files.sh cabal-install.cabal)
+
+# Fail if the diff is not empty.
+timed ./Cabal/misc/travis-diff-files.sh