Add test-wrapper option to wrap test execution

Cabal/Distribution/Simple/Setup.hs

@@ -1864,6 +1864,7 @@ data TestFlags = TestFlags {
     testMachineLog  :: Flag PathTemplate,
     testShowDetails :: Flag TestShowDetails,
     testKeepTix     :: Flag Bool,
+    testWrapper     :: Flag FilePath,
     testFailWhenNoTestSuites :: Flag Bool,
     -- TODO: think about if/how options are passed to test exes
     testOptions     :: [PathTemplate]
@@ -1877,6 +1878,7 @@ defaultTestFlags  = TestFlags {
     testMachineLog  = toFlag $ toPathTemplate $ "$pkgid.log",
     testShowDetails = toFlag Failures,
     testKeepTix     = toFlag False,
+    testWrapper     = NoFlag,
     testFailWhenNoTestSuites = toFlag False,
     testOptions     = []
   }
@@ -1942,6 +1944,11 @@ testOptions' showOrParseArgs =
         "keep .tix files for HPC between test runs"
         testKeepTix (\v flags -> flags { testKeepTix = v})
         trueArg
+  , option [] ["test-wrapper"]
+        "Run test through a wrapper."
+        testWrapper (\v flags -> flags { testWrapper = v })
+        (reqArg' "FILE" (toFlag :: FilePath -> Flag FilePath)
+            (flagToList :: Flag FilePath -> [FilePath]))
   , option [] ["fail-when-no-test-suites"]
         ("Exit with failure when no test suites are found.")
         testFailWhenNoTestSuites (\v flags -> flags { testFailWhenNoTestSuites = v})

Cabal/Distribution/Simple/Test/ExeV10.hs

@@ -98,7 +98,12 @@ runTest pkg_descr lbi clbi flags suite = do
                             return (addLibraryPath os paths shellEnv)
                     else return shellEnv
 
-    exit <- rawSystemIOWithEnv verbosity cmd opts Nothing (Just shellEnv')
+    exit <- case testWrapper flags of
+      Flag path -> rawSystemIOWithEnv verbosity path (cmd:opts) Nothing (Just shellEnv')
+                               -- these handles are automatically closed
+                               Nothing (Just wOut) (Just wErr)
+
+      NoFlag -> rawSystemIOWithEnv verbosity cmd opts Nothing (Just shellEnv')
                                -- these handles are automatically closed
                                Nothing (Just wOut) (Just wErr)
 

Cabal/Distribution/Simple/Test/LibV09.hs

@@ -99,9 +99,14 @@ runTest pkg_descr lbi clbi flags suite = do
                     cpath <- canonicalizePath $ LBI.componentBuildDir lbi clbi
                     return (addLibraryPath os (cpath : paths) shellEnv)
                   else return shellEnv
-                createProcessWithEnv verbosity cmd opts Nothing (Just shellEnv')
-                                     -- these handles are closed automatically
-                                     CreatePipe (UseHandle wOut) (UseHandle wOut)
+                case testWrapper flags of
+                  Flag path -> createProcessWithEnv verbosity path (cmd:opts) Nothing (Just shellEnv')
+                               -- these handles are closed automatically
+                               CreatePipe (UseHandle wOut) (UseHandle wOut)
+
+                  NoFlag -> createProcessWithEnv verbosity cmd opts Nothing (Just shellEnv')
+                            -- these handles are closed automatically
+                            CreatePipe (UseHandle wOut) (UseHandle wOut)
 
         hPutStr wIn $ show (tempLog, PD.testName suite)
         hClose wIn

Cabal/doc/installing-packages.rst

@@ -1719,6 +1719,13 @@ the package.
     quote options containing spaces because a single option is assumed,
     so options will not be split on spaces.
 
+.. option:: --test-wrapper=path
+
+   The wrapper script/application used to setup and tear down the test
+   execution context. The text executable path and test arguments are
+   passed as arguments to the wrapper and it is expected that the wrapper
+   will return the test's return code, as well as a copy of stdout/stderr.
+
 .. _setup-sdist:
 
 setup sdist

cabal-install/Distribution/Client/Config.hs

@@ -500,6 +500,7 @@ instance Semigroup SavedConfig where
         testMachineLog  = combine testMachineLog,
         testShowDetails = combine testShowDetails,
         testKeepTix     = combine testKeepTix,
+        testWrapper     = combine testWrapper,
         testFailWhenNoTestSuites = combine testFailWhenNoTestSuites,
         testOptions     = lastNonEmpty testOptions
         }

cabal-install/Distribution/Client/Install.hs

@@ -78,7 +78,8 @@ import Distribution.Client.SolverInstallPlan (SolverInstallPlan)
 import Distribution.Client.Setup
          ( GlobalFlags(..), RepoContext(..)
          , ConfigFlags(..), configureCommand, filterConfigureFlags
-         , ConfigExFlags(..), InstallFlags(..) )
+         , ConfigExFlags(..), InstallFlags(..)
+         , filterTestFlags )
 import Distribution.Client.Config
          ( getCabalDir, defaultUserInstall )
 import Distribution.Client.Sandbox.Timestamp
@@ -1490,7 +1491,7 @@ installUnpackedPackage verbosity installLock numJobs
     }
     testsEnabled = fromFlag (configTests configFlags)
                    && fromFlagOrDefault False (installRunTests installFlags)
-    testFlags' _ = testFlags {
+    testFlags' = filterTestFlags testFlags {
       Cabal.testDistPref = configDistPref configFlags
     }
     copyFlags _ = Cabal.emptyCopyFlags {

cabal-install/Distribution/Client/ProjectBuilding.hs

@@ -65,7 +65,7 @@ import           Distribution.Client.GlobalFlags (RepoContext)
 import qualified Distribution.Client.Tar as Tar
 import           Distribution.Client.Setup
                    ( filterConfigureFlags, filterHaddockArgs
-                   , filterHaddockFlags )
+                   , filterHaddockFlags, filterTestFlags )
 import           Distribution.Client.SourceFiles
 import           Distribution.Client.SrcDist (allPackageSourceFiles)
 import           Distribution.Client.Utils
@@ -1370,7 +1370,8 @@ buildInplaceUnpackedPackage verbosity
     buildArgs     _  = setupHsBuildArgs  pkg
 
     testCommand      = Cabal.testCommand -- defaultProgramDb
-    testFlags    _   = setupHsTestFlags pkg pkgshared
+    testFlags      v = flip filterTestFlags v $
+                       setupHsTestFlags pkg pkgshared
                                          verbosity builddir
     testArgs      _  = setupHsTestArgs  pkg
 

cabal-install/Distribution/Client/ProjectConfig/Legacy.hs

@@ -450,6 +450,7 @@ convertLegacyPerPackageFlags configFlags installFlags haddockFlags testFlags =
       testMachineLog            = packageConfigTestMachineLog,
       testShowDetails           = packageConfigTestShowDetails,
       testKeepTix               = packageConfigTestKeepTix,
+      testWrapper               = packageConfigTestWrapper,
       testFailWhenNoTestSuites  = packageConfigTestFailWhenNoTestSuites,
       testOptions               = packageConfigTestTestOptions
     } = testFlags
@@ -795,6 +796,7 @@ convertToLegacyPerPackageConfig PackageConfig {..} =
       testMachineLog  = packageConfigTestMachineLog,
       testShowDetails = packageConfigTestShowDetails,
       testKeepTix     = packageConfigTestKeepTix,
+      testWrapper     = packageConfigTestWrapper,
       testFailWhenNoTestSuites = packageConfigTestFailWhenNoTestSuites,
       testOptions     = packageConfigTestTestOptions
     }
@@ -1090,7 +1092,8 @@ legacyPackageConfigFieldDescrs =
           (\v conf -> conf { testOptions = v })
       ]
   . filterFields
-      [ "log", "machine-log", "show-details", "keep-tix-files", "fail-when-no-test-suites" ]
+      [ "log", "machine-log", "show-details", "keep-tix-files"
+      , "fail-when-no-test-suites", "test-wrapper" ]
   . commandOptionsToFields
   ) (testOptions' ParseArgs)
 

cabal-install/Distribution/Client/ProjectConfig/Types.hs

@@ -291,6 +291,7 @@ data PackageConfig
        packageConfigTestMachineLog      :: Flag PathTemplate,
        packageConfigTestShowDetails     :: Flag TestShowDetails,
        packageConfigTestKeepTix         :: Flag Bool,
+       packageConfigTestWrapper         :: Flag FilePath,
        packageConfigTestFailWhenNoTestSuites :: Flag Bool,
        packageConfigTestTestOptions     :: [PathTemplate]
      }

cabal-install/Distribution/Client/ProjectPlanning.hs

@@ -1884,6 +1884,7 @@ elaborateInstallPlan verbosity platform compiler compilerprogdb pkgConfigDB
         elabTestHumanLog        = perPkgOptionMaybe pkgid packageConfigTestHumanLog
         elabTestShowDetails     = perPkgOptionMaybe pkgid packageConfigTestShowDetails
         elabTestKeepTix         = perPkgOptionFlag pkgid False packageConfigTestKeepTix
+        elabTestWrapper         = perPkgOptionMaybe pkgid packageConfigTestWrapper
         elabTestFailWhenNoTestSuites = perPkgOptionFlag pkgid False packageConfigTestFailWhenNoTestSuites
         elabTestTestOptions     = perPkgOptionList pkgid packageConfigTestTestOptions
 
@@ -3440,6 +3441,7 @@ setupHsTestFlags (ElaboratedConfiguredPackage{..}) _ verbosity builddir = Cabal.
     , testHumanLog    = maybe mempty toFlag elabTestHumanLog
     , testShowDetails = maybe (Flag Cabal.Always) toFlag elabTestShowDetails
     , testKeepTix     = toFlag elabTestKeepTix
+    , testWrapper     = maybe mempty toFlag elabTestWrapper
     , testFailWhenNoTestSuites = toFlag elabTestFailWhenNoTestSuites
     , testOptions     = elabTestTestOptions
     }

cabal-install/Distribution/Client/ProjectPlanning/Types.hs

@@ -294,6 +294,7 @@ data ElaboratedConfiguredPackage
        elabTestHumanLog          :: Maybe PathTemplate,
        elabTestShowDetails       :: Maybe TestShowDetails,
        elabTestKeepTix           :: Bool,
+       elabTestWrapper           :: Maybe FilePath,
        elabTestFailWhenNoTestSuites :: Bool,
        elabTestTestOptions       :: [PathTemplate],
 

cabal-install/Distribution/Client/Setup.hs

@@ -22,6 +22,7 @@ module Distribution.Client.Setup
     , configPackageDB', configCompilerAux'
     , configureExCommand, ConfigExFlags(..), defaultConfigExFlags
     , buildCommand, BuildFlags(..), BuildExFlags(..), SkipAddSourceDepsCheck(..)
+    , filterTestFlags
     , replCommand, testCommand, benchmarkCommand, testOptions
                         , configureExOptions, reconfigureCommand
     , installCommand, InstallFlags(..), installOptions, defaultInstallFlags
@@ -829,6 +830,37 @@ instance Monoid BuildExFlags where
 instance Semigroup BuildExFlags where
   (<>) = gmappend
 
+-- ------------------------------------------------------------
+-- * Test flags
+-- ------------------------------------------------------------
+
+-- | Given some 'TestFlags' for the version of Cabal that
+-- cabal-install was built with, and a target older 'Version' of
+-- Cabal that we want to pass these flags to, convert the
+-- flags into a form that will be accepted by the older
+-- Setup script.  Generally speaking, this just means filtering
+-- out flags that the old Cabal library doesn't understand, but
+-- in some cases it may also mean "emulating" a feature using
+-- some more legacy flags.
+filterTestFlags :: TestFlags -> Version -> TestFlags
+filterTestFlags flags cabalLibVersion
+  -- NB: we expect the latest version to be the most common case,
+  -- so test it first.
+  | cabalLibVersion >= mkVersion [3,0,0] = flags_latest
+  -- The naming convention is that flags_version gives flags with
+  -- all flags *introduced* in version eliminated.
+  -- It is NOT the latest version of Cabal library that
+  -- these flags work for; version of introduction is a more
+  -- natural metric.
+  | cabalLibVersion <  mkVersion [3,0,0] = flags_3_0_0
+  | otherwise = error "the impossible just happened" -- see first guard
+  where
+    flags_latest = flags
+    flags_3_0_0  = flags_latest {
+      -- Cabal < 3.0 doesn't know about --test-wrapper
+      Cabal.testWrapper = NoFlag
+      }
+
 -- ------------------------------------------------------------
 -- * Repl command
 -- ------------------------------------------------------------
@@ -1929,7 +1961,8 @@ testOptions showOrParseArgs
     | opt <- commandOptions Cabal.testCommand showOrParseArgs
     , let name = optionName opt
     , name `elem` ["log", "machine-log", "show-details", "keep-tix-files"
-                  ,"fail-when-no-test-suites", "test-options", "test-option"]
+                  ,"fail-when-no-test-suites", "test-options", "test-option"
+                  ,"test-wrapper"]
     ]
   where
     prefixTest name | "test-" `isPrefixOf` name = name

cabal-install/changelog

@@ -27,6 +27,8 @@
 	  that make it possible to copy the executable instead of symlinking it
 	* --symlink-bindir no longer controls the symlinking directory of
 	  v2-install (installdir controls both symlinking and copying now)
+	* Add --test-wrapper that allows a prebuild script to set the test environment.
+	* Add filterTestFlags: filter test-wrapper for Cabal < 3.0.0.
 
 2.4.1.0 Mikhail Glushenkov <[email protected]> November 2018
 	* Add message to alert user to potential package casing errors. (#5635)

cabal-install/tests/UnitTests/Distribution/Client/ProjectConfig.hs

@@ -597,6 +597,7 @@ instance Arbitrary PackageConfig where
         <*> arbitrary
         <*> arbitrary
         <*> arbitrary
+        <*> arbitraryFlag arbitraryShortToken
         <*> arbitrary
         <*> shortListOf 5 arbitrary
       where
@@ -656,8 +657,9 @@ instance Arbitrary PackageConfig where
                          , packageConfigTestMachineLog = x45
                          , packageConfigTestShowDetails = x46
                          , packageConfigTestKeepTix = x47
-                         , packageConfigTestFailWhenNoTestSuites = x48
-                         , packageConfigTestTestOptions = x49 } =
+                         , packageConfigTestWrapper = x48
+                         , packageConfigTestFailWhenNoTestSuites = x49
+                         , packageConfigTestTestOptions = x51 } =
       [ PackageConfig { packageConfigProgramPaths = postShrink_Paths x00'
                       , packageConfigProgramArgs = postShrink_Args x01'
                       , packageConfigProgramPathExtra = x02'
@@ -709,8 +711,9 @@ instance Arbitrary PackageConfig where
                       , packageConfigTestMachineLog = x45'
                       , packageConfigTestShowDetails = x46'
                       , packageConfigTestKeepTix = x47'
-                      , packageConfigTestFailWhenNoTestSuites = x48'
-                      , packageConfigTestTestOptions = x49' }
+                      , packageConfigTestWrapper = x48'
+                      , packageConfigTestFailWhenNoTestSuites = x49'
+                      , packageConfigTestTestOptions = x51' }
       |  (((x00', x01', x02', x03', x04'),
           (x05', x42', x06', x50', x07', x08', x09'),
           (x10', x11', x12', x13', x14'),
@@ -720,7 +723,7 @@ instance Arbitrary PackageConfig where
           (x30', x31', x32', (x33', x33_1'), x34'),
           (x35', x36', x37', x38', x43', x39'),
           (x40', x41'),
-          (x44', x45', x46', x47', x48', x49')))
+          (x44', x45', x46', x47', x48', x49', x51')))
           <- shrink
              (((preShrink_Paths x00, preShrink_Args x01, x02, x03, x04),
                 (x05, x42, x06, x50, x07, x08, x09),
@@ -734,7 +737,7 @@ instance Arbitrary PackageConfig where
                  (x30, x31, x32, (x33, x33_1), x34),
                  (x35, x36, fmap NonEmpty x37, x38, x43, fmap NonEmpty x39),
                  (x40, x41),
-                 (x44, x45, x46, x47, x48, x49)))
+                 (x44, x45, x46, x47, x48, x49, x51)))
       ]
       where
         preShrink_Paths  = Map.map NonEmpty