Use Cabal directly in place of ghc-cabal + make build root configurable

.travis.yml

@@ -2,61 +2,91 @@ sudo: true
 matrix:
     include:
         - os: linux
-          env: MODE="--flavour=quickest inplace/bin/ghc-stage1"
-          compiler: "GHC 8.0.2"
+          env: MODE="selftest"
+          compiler: "GHC 8.2.2"
           addons:
               apt:
                   packages:
-                      - ghc-8.0.2
+                      - ghc-8.2.2
                       - cabal-install-2.0
                       - zlib1g-dev
                   sources: hvr-ghc
 
           before_install:
-              - PATH="/opt/ghc/8.0.2/bin:$PATH"
+              - PATH="/opt/ghc/8.2.2/bin:$PATH"
               - PATH="/opt/cabal/2.0/bin:$PATH"
 
           script:
+              # boot & configure ghc source tree
+              - ./boot && ./configure
               # Run internal Hadrian tests
-              - ./build.sh -c selftest
+              - hadrian/build.sh selftest
 
-              # Build GHC
-              - ./build.sh -j -c $MODE --no-progress --progress-colour=never --profile=-
+        - os: linux
+          env: MODE="--flavour=quickest"
+          compiler: "GHC 8.2.2"
+          addons:
+              apt:
+                  packages:
+                      - ghc-8.2.2
+                      - cabal-install-2.0
+                      - zlib1g-dev
+                  sources: hvr-ghc
+
+          before_install:
+              - PATH="/opt/ghc/8.2.2/bin:$PATH"
+              - PATH="/opt/cabal/2.0/bin:$PATH"
+
+          script:
+              # boot & configure ghc source tree
+              - ./boot && ./configure
+
+              # Build GHC, letting hadrian boot & configure the ghc source tree
+              - hadrian/build.sh -j $MODE --no-progress --progress-colour=never --profile=-
 
         - os: linux
           env: MODE="--flavour=quickest --integer-simple"
-          compiler: "GHC 8.2.1"
+          compiler: "GHC 8.4.1"
           addons:
               apt:
                   packages:
-                      - ghc-8.2.1
-                      - cabal-install-1.22
+                      - ghc-8.4.1
+                      - cabal-install-2.2
                       - zlib1g-dev
                   sources: hvr-ghc
 
           before_install:
-              - PATH="/opt/ghc/8.2.1/bin:$PATH"
-              - PATH="/opt/cabal/1.22/bin:$PATH"
+              - PATH="/opt/ghc/8.4.1/bin:$PATH"
+              - PATH="/opt/cabal/2.2/bin:$PATH"
 
           script:
-              # Build GHC
-              - ./build.sh -j -c $MODE --no-progress --progress-colour=never --profile=-
+              # boot & configure ghc source tree
+              - ./boot && ./configure
+
+              # build GHC
+              - hadrian/build.sh -j $MODE --no-progress --progress-colour=never --profile=-
 
               # Test GHC binary
-              - cd ..
-              - inplace/bin/ghc-stage2 -e 1+2
+              - _build/stage1/bin/ghc -e 1+2
 
         - os: osx
           osx_image: xcode8
-          env: MODE="--flavour=quickest --integer-simple inplace/bin/ghc-stage1"
+          env: MODE="--flavour=quickest --integer-simple"
           before_install:
               - brew update
-              - brew install ghc cabal-install python3
+              - brew install ghc cabal-install
+              - brew upgrade python
 
           script:
+              # boot and configure ghc source tree
+              - ./boot && ./configure
+
               # Due to timeout limit of OS X build on Travis CI,
               # we will ignore selftest and build only stage1
-              - ./build.sh -j -c $MODE --no-progress --progress-colour=never --profile=-
+              - hadrian/build.sh -j -c $MODE --no-progress --progress-colour=never --profile=-
+
+              # Test GHC binary
+              - _build/stage1/bin/ghc -e 1+2
 
 install:
     # Add Cabal to PATH
@@ -83,8 +113,7 @@ install:
     # to ./ghc/hadrian -- one way to do it is to move the .git directory
     # and perform a hard reset in order to regenerate Hadrian files
     - mv .git ghc/hadrian
-    - cd ghc/hadrian
-    - git reset --hard HEAD
+    - cd ghc/hadrian && git reset --hard HEAD && cd ..
 
 cache:
     directories:

README.md

@@ -57,6 +57,14 @@ are placed into `_build` and `inplace` directories.
 In addition to standard Shake flags (try `--help`), the build system
 currently supports several others:
 
+* `--build-root=PATH` or `-oPATH`: specify the directory in which you want to store all
+the build artifacts. If none is specified by the user, hadrian will store everything
+under `_build/` at the top of ghc's source tree. Unlike GHC's make build system,
+hadrian doesn't have any "inplace" logic left anymore. This option is therefore
+useful for GHC developers who want to build GHC in different ways or at different
+commits, from the same directory, and have the build products sit in different,
+isolated folders.
+
 * `--configure` or `-c`: use this flag to run the `boot` and `configure` scripts
 automatically, so that you don't have to remember to run them manually as you normally
 do when using Make (typically only in the first build):
@@ -119,17 +127,6 @@ are currently not supported.
 
 To build a GHC source distribution tarball, run `build sdist-ghc`.
 
-#### Installation
-
-To build and install GHC artifacts, run `build install`.
-
-By default, GHC will be installed to the specified _prefix_ path on your system,
-relative to the root of the file system. For example on UNIX, GHC will be installed
-to `/usr/local/bin`. By setting the command line flag `--install-destdir=[DESTDIR]`,
-you can install GHC to path `DESTDIR/<prefix>` instead. Make sure you use correct
-absolute path as `DESTDIR` on Windows, e.g. `C:/path`, which installs GHC
-into `C:/path/usr/local`.
-
 #### Testing
 
 * `build validate` runs GHC tests by simply executing `make fast` in `testsuite/tests`

appveyor.yml

@@ -30,12 +30,17 @@ install:
     - appveyor-retry stack exec -- pacman -S autoconf automake-wrapper make patch python tar --noconfirm
 
 build_script:
+    # Boot and configure ghc source tree
+    - cd ..
+    - python boot && configure --enable-tarballs-autodownload
+    - cd hadrian
+
     # Build Hadrian and run internal Hadrian tests
-    - build -c selftest
+    - build selftest
 
     # Build GHC
-    - build -j -c --flavour=quickest --no-progress --progress-colour=never --profile=-
+    - build -j --flavour=quickest --no-progress --progress-colour=never --profile=-
 
     # Test GHC binary
     - cd ..
-    - inplace\bin\ghc-stage2 -e 1+2
+    - _build/stage1/bin/ghc -e 1+2

doc/user-settings.md

@@ -5,16 +5,6 @@ You can customise Hadrian by copying the file `hadrian/src/UserSettings.hs` to
 copy the file your changes will be tracked by `git` and you can accidentally commit
 them). Here we document currently supported settings.
 
-## Build directory
-
-Hadrian puts build results into `_build` directory by default, which is
-specified by `userBuildRoot`:
-```haskell
--- | All build results are put into the 'buildRoot' directory.
-userBuildRoot :: BuildRoot
-userBuildRoot = BuildRoot "_build"
-```
-
 ## Build flavour
 
 Build _flavour_ is a collection of build settings that fully define a GHC build

hadrian.cabal

@@ -22,18 +22,21 @@ executable hadrian
                        , Builder
                        , CommandLine
                        , Context
+                       , Context.Paths
                        , Context.Type
                        , Environment
                        , Expression
                        , Expression.Type
                        , Flavour
                        , GHC
+                       , GHC.Packages
                        , Hadrian.Builder
                        , Hadrian.Builder.Ar
                        , Hadrian.Builder.Sphinx
                        , Hadrian.Builder.Tar
                        , Hadrian.Expression
                        , Hadrian.Haskell.Cabal
+                       , Hadrian.Haskell.Cabal.PackageData
                        , Hadrian.Haskell.Cabal.Parse
                        , Hadrian.Haskell.Cabal.Type
                        , Hadrian.Oracles.ArgsHash
@@ -47,7 +50,6 @@ executable hadrian
                        , Oracles.Flag
                        , Oracles.Setting
                        , Oracles.ModuleFiles
-                       , Oracles.PackageData
                        , Rules
                        , Rules.Clean
                        , Rules.Compile
@@ -57,15 +59,13 @@ executable hadrian
                        , Rules.Documentation
                        , Rules.Generate
                        , Rules.Gmp
-                       , Rules.Install
                        , Rules.Libffi
                        , Rules.Library
                        , Rules.Program
                        , Rules.Register
                        , Rules.Selftest
                        , Rules.SourceDist
                        , Rules.Test
-                       , Rules.Wrappers
                        , Settings
                        , Settings.Builders.Alex
                        , Settings.Builders.Common
@@ -91,19 +91,8 @@ executable hadrian
                        , Settings.Flavours.Quick
                        , Settings.Flavours.QuickCross
                        , Settings.Flavours.Quickest
-                       , Settings.Packages.Base
-                       , Settings.Packages.Cabal
-                       , Settings.Packages.Compiler
-                       , Settings.Packages.Ghc
-                       , Settings.Packages.GhcCabal
-                       , Settings.Packages.Ghci
-                       , Settings.Packages.GhcPkg
-                       , Settings.Packages.GhcPrim
-                       , Settings.Packages.Haddock
-                       , Settings.Packages.Haskeline
-                       , Settings.Packages.IntegerGmp
+                       , Settings.Packages
                        , Settings.Packages.Rts
-                       , Settings.Packages.RunGhc
                        , Settings.Warnings
                        , Stage
                        , Target

src/Base.hs

@@ -18,12 +18,14 @@ module Base (
     module Stage,
     module Way,
 
+    -- * Files
+    configH, ghcVersionH,
     -- * Paths
-    hadrianPath, configPath, configFile, sourcePath, configH, shakeFilesDir,
-    generatedDir, inplaceBinPath, inplaceLibBinPath, inplaceLibPath,
-    inplaceLibCopyTargets, haddockHtmlResourcesStamp, templateHscPath,
-    stage0PackageDbDir, inplacePackageDbPath, packageDbPath, packageDbStamp,
-    ghcSplitPath
+    hadrianPath, configPath, configFile, sourcePath, shakeFilesDir,
+    generatedDir, generatedPath,
+    stageBinPath, stageLibPath,
+    templateHscPath, ghcDeps,
+    relativePackageDbPath, packageDbPath, packageDbStamp, ghcSplitPath
     ) where
 
 import Control.Applicative
@@ -65,71 +67,60 @@ sourcePath = hadrianPath -/- "src"
 configH :: FilePath
 configH = "mk/config.h"
 
+ghcVersionH :: Action FilePath
+ghcVersionH = generatedPath <&> (-/- "ghcversion.h")
+
 -- | The directory in 'buildRoot' containing the Shake database and other
 -- auxiliary files generated by Hadrian.
 shakeFilesDir :: FilePath
 shakeFilesDir = "hadrian"
 
--- | Directory for binaries that are built "in place".
-inplaceBinPath :: FilePath
-inplaceBinPath = "inplace/bin"
-
--- | Directory for libraries that are built "in place".
-inplaceLibPath :: FilePath
-inplaceLibPath = "inplace/lib"
-
--- | Directory for binary wrappers, and auxiliary binaries such as @touchy@.
-inplaceLibBinPath :: FilePath
-inplaceLibBinPath = inplaceLibPath -/- "bin"
-
 -- | The directory in 'buildRoot' containing generated source files that are not
 -- package-specific, e.g. @ghcplatform.h@.
 generatedDir :: FilePath
 generatedDir = "generated"
 
--- | The directory in 'buildRoot' containing the 'Stage0' package database.
-stage0PackageDbDir :: FilePath
-stage0PackageDbDir = "stage0/bootstrapping.conf"
+generatedPath :: Action FilePath
+generatedPath = buildRoot <&> (-/- generatedDir)
 
--- | Path to the inplace package database used in 'Stage1' and later.
-inplacePackageDbPath :: FilePath
-inplacePackageDbPath = inplaceLibPath -/- "package.conf.d"
+-- | Path to the package database for the given stage of GHC,
+--   relative to the build root.
+relativePackageDbPath :: Stage -> FilePath
+relativePackageDbPath stage = stageString stage -/- "lib" -/- "package.conf.d"
 
--- | Path to the package database used in a given 'Stage'.
+-- | Path to the package database used in a given 'Stage', including
+--   the build root.
 packageDbPath :: Stage -> Action FilePath
-packageDbPath Stage0 = buildRoot <&> (-/- stage0PackageDbDir)
-packageDbPath _      = return inplacePackageDbPath
+packageDbPath stage = buildRoot <&> (-/- relativePackageDbPath stage)
 
 -- | We use a stamp file to track the existence of a package database.
 packageDbStamp :: FilePath
 packageDbStamp = ".stamp"
 
--- ref: GHC_DEPENDENCIES in ghc/ghc.mk
--- ref: INSTALL_LIBS in driver/ghc.mk
--- TODO: Derive this from Builder.runtimeDependencies
--- | Files that need to be copied over to 'inplaceLibPath'.
-inplaceLibCopyTargets :: [FilePath]
-inplaceLibCopyTargets = map (inplaceLibPath -/-)
-    [ "ghc-usage.txt"
-    , "ghci-usage.txt"
-    , "llvm-targets"
-    , "platformConstants"
-    , "settings"
-    , "template-hsc.h" ]
-
--- TODO: This is fragile and will break if @README.md@ is removed. We need to
--- improve the story of program runtime dependencies on directories.
--- See: https://github.com/snowleopard/hadrian/issues/492.
--- | Path to a file in Haddock's HTML resource library.
-haddockHtmlResourcesStamp :: FilePath
-haddockHtmlResourcesStamp = inplaceLibPath -/- "html/README.md"
+-- | @bin@ directory for the given 'Stage' (including the build root)
+stageBinPath :: Stage -> Action FilePath
+stageBinPath stage = buildRoot <&> (-/- stageString stage -/- "bin")
+
+-- | @lib@ directory for the given 'Stage' (including the build root)
+stageLibPath :: Stage -> Action FilePath
+stageLibPath stage = buildRoot <&> (-/- stageString stage -/- "lib")
+
+-- | Files the `ghc` binary depends on
+ghcDeps :: Stage -> Action [FilePath]
+ghcDeps stage = mapM (\f -> stageLibPath stage <&> (-/- f))
+      [ "ghc-usage.txt"
+      , "ghci-usage.txt"
+      , "llvm-targets"
+      , "platformConstants"
+      , "settings" ]
 
 -- ref: utils/hsc2hs/ghc.mk
 -- | Path to 'hsc2hs' template.
-templateHscPath :: FilePath
-templateHscPath = inplaceLibPath -/- "template-hsc.h"
+templateHscPath :: Stage -> Action FilePath
+templateHscPath stage = stageLibPath stage <&> (-/- "template-hsc.h")
 
 -- | @ghc-split@ is a Perl script used by GHC when run with @-split-objs@ flag.
--- It is generated in "Rules.Generate".
-ghcSplitPath :: FilePath
-ghcSplitPath = inplaceLibBinPath -/- "ghc-split"
+-- It is generated in "Rules.Generate". This function returns the path relative
+-- to the build root under which we will copy @ghc-split@.
+ghcSplitPath :: Stage -> FilePath
+ghcSplitPath stage = stageString stage -/- "bin" -/- "ghc-split"