Mark failing tests as expected failures, or fix them

cabal-install/cabal-install.cabal

@@ -339,7 +339,7 @@ Test-Suite memory-usage-tests
   hs-source-dirs: tests
   default-language: Haskell2010
 
-  ghc-options: -threaded -rtsopts "-with-rtsopts=-M8M -K1K"
+  ghc-options: -threaded -rtsopts "-with-rtsopts=-M16M -K1K"
 
   other-modules:
     UnitTests.Distribution.Solver.Modular.DSL

cabal-testsuite/PackageTests/Backpack/Includes2/setup-external.test.hs

@@ -1,6 +1,8 @@
 import Test.Cabal.Prelude
 main = setupAndCabalTest $ do
-    skipUnlessGhcVersion ">= 8.1"
+  skipUnlessGhcVersion ">= 8.1"
+  ghc <- isGhcVersion "== 9.0.2 || == 9.2.1"
+  expectBrokenIf ghc 7987 $ do
     withPackageDb $ do
       withDirectory "mylib" $ setup_install_with_docs ["--ipid", "mylib-0.1.0.0"]
       withDirectory "mysql" $ setup_install_with_docs ["--ipid", "mysql-0.1.0.0"]

cabal-testsuite/PackageTests/Backpack/Includes2/setup-per-component.test.hs

@@ -1,7 +1,9 @@
 import Test.Cabal.Prelude
 main = setupTest $ do
-    -- No cabal test because per-component is broken with it
-    skipUnlessGhcVersion ">= 8.1"
+  -- No cabal test because per-component is broken with it
+  skipUnlessGhcVersion ">= 8.1"
+  ghc <- isGhcVersion "== 9.0.2 || == 9.2.1"
+  expectBrokenIf ghc 7987 $
     withPackageDb $ do
       let setup_install' args = setup_install_with_docs (["--cabal-file", "Includes2.cabal"] ++ args)
       setup_install' ["mylib", "--cid", "mylib-0.1.0.0"]

cabal-testsuite/PackageTests/Backpack/Includes3/setup-external-ok.test.hs

@@ -2,7 +2,9 @@ import Test.Cabal.Prelude
 import Data.List
 import qualified Data.Char as Char
 main = setupAndCabalTest $ do
-    skipUnlessGhcVersion ">= 8.1"
+  skipUnlessGhcVersion ">= 8.1"
+  ghc <- isGhcVersion "== 9.0.2 || == 9.2.1"
+  expectBrokenIf ghc 7987 $
     withPackageDb $ do
       containers_id <- getIPID "containers"
       withDirectory "repo/sigs-0.1.0.0" $ setup_install_with_docs ["--ipid", "sigs-0.1.0.0"]

cabal-testsuite/PackageTests/CmmSourcesDyn/setup.test.hs

@@ -1,7 +1,7 @@
 import Test.Cabal.Prelude
 
 main = setupTest $ do
-    skipIf "ghc < 7.8" =<< ghcVersionIs (< mkVersion [7,8])
+    skipIf "ghc < 7.8" =<< isGhcVersion "< 7.8"
     setup "configure" []
     res <- setup' "build" []
     assertOutputContains "= Post common block elimination =" res

cabal-testsuite/PackageTests/CustomPlain/setup.test.hs

@@ -1,10 +1,5 @@
 import Test.Cabal.Prelude
 main = setupTest $ do
     skipUnless "no Cabal for GHC" =<< hasCabalForGhc
-    -- On Travis OSX, Cabal shipped with GHC 7.8 does not work
-    -- with error "setup: /usr/bin/ar: permission denied"; see
-    -- also https://github.com/haskell/cabal/issues/3938
-    -- This is a hack to make the test not run in this case.
-    skipIf "osx and ghc < 7.10" =<< liftM2 (&&) isOSX (ghcVersionIs (< mkVersion [7,10]))
     setup' "configure" [] >>= assertOutputContains "ThisIsCustomYeah"
     setup' "build"     [] >>= assertOutputContains "ThisIsCustomYeah"

cabal-testsuite/PackageTests/ForeignLibs/setup.test.hs

@@ -24,8 +24,11 @@ import Test.Cabal.Prelude
 -- Recording is turned off because versionedlib will or will not
 -- be installed depending on if we're on Linux or not.
 main = setupAndCabalTest . recordMode DoNotRecord $ do
-    -- Foreign libraries don't work with GHC 7.6 and earlier
-    skipUnlessGhcVersion ">= 7.8"
+  -- Foreign libraries don't work with GHC 7.6 and earlier
+  skipUnlessGhcVersion ">= 7.8"
+  osx <- isOSX
+  ghc <- isGhcVersion "== 8.0.2"
+  expectBrokenIf (osx && ghc) 7989 $
     withPackageDb $ do
         setup_install []
         setup "copy" [] -- regression test #4156

cabal-testsuite/PackageTests/NewBuild/T3827/cabal.test.hs

@@ -1,3 +1,7 @@
 import Test.Cabal.Prelude
 main = cabalTest $ do
+  missesProfiling <- isGhcVersion ">= 9.2.1"
+  osx <- isOSX
+  missesProfilingOsx <- isGhcVersion ">= 8.10.7"
+  expectBrokenIf (missesProfiling || osx && missesProfilingOsx) 8032 $
     cabal "v2-build" ["exe:q"]

cabal-testsuite/PackageTests/Regression/T4025/setup.test.hs

@@ -3,7 +3,10 @@ import Test.Cabal.Prelude
 -- an executable RPATH.  Don't test on Windows, which doesn't
 -- support RPATH.
 main = setupAndCabalTest $ do
-    skipIfWindows
+  skipIfWindows
+  osx <- isOSX
+  ghc <- isGhcVersion ">= 8.10.7"
+  expectBrokenIf (osx && ghc) 7610 $ do -- see also issue #7988
     setup "configure" ["--enable-executable-dynamic"]
     setup "build" []
     -- This should fail as it we should NOT be able to find the

cabal-testsuite/PackageTests/Regression/T4270/setup.test.hs

@@ -6,5 +6,8 @@ main = setupAndCabalTest $ do
   skipUnless "no shared libs"   =<< hasSharedLibraries
   skipUnless "no shared Cabal"  =<< hasCabalShared
   skipUnless "no Cabal for GHC" =<< hasCabalForGhc
-  setup_build ["--enable-tests", "--enable-executable-dynamic"]
-  setup "test" []
+  ghc <- isGhcVersion "== 8.0.2"
+  osx <- isOSX
+  expectBrokenIf (osx && ghc) 8028 $ do
+    setup_build ["--enable-tests", "--enable-executable-dynamic"]
+    setup "test" []

cabal-testsuite/PackageTests/SPDX/cabal-old-build.test.hs

@@ -2,6 +2,6 @@ import Test.Cabal.Prelude
 main = setupAndCabalTest $ withPackageDb $ do
     setup_install []
     recordMode DoNotRecord $ do
-        ghc84 <- ghcVersionIs (>= mkVersion [8,4])
+        ghc84 <- isGhcVersion ">= 8.4"
         let lic = if ghc84 then "BSD-3-Clause" else "BSD3"
         ghcPkg' "field" ["my", "license"] >>= assertOutputContains lic

cabal-testsuite/PackageTests/TestSuiteTests/ExeV10/cabal-with-hpc.multitest.hs

@@ -9,7 +9,7 @@ import qualified Distribution.Verbosity as Verbosity
 import Test.Cabal.Prelude
 
 main = cabalTest $ do
-    skipIf "osx" =<< isOSX -- TODO: re-enable this once the macOS Travis
+    skipIf "osx" =<< isOSX -- TODO: re-enable this once the macOS CI
                            -- issues are resolved, see discussion in #4902.
 
     hasShared   <- hasSharedLibraries

cabal-testsuite/README.md

@@ -135,7 +135,7 @@ and stderr.
 **How do I skip running a test in some environments?**  Use the
 `skipIf` and `skipUnless` combinators.  Useful parameters to test
 these with include `hasSharedLibraries`, `hasProfiledLibraries`,
-`hasCabalShared`, `ghcVersionIs`, `isWindows`, `isLinux`, `isOSX`
+`hasCabalShared`, `isGhcVersion`, `isWindows`, `isLinux`, `isOSX`
 and `hasCabalForGhc`.
 
 **I programatically modified a file in my test suite, but Cabal/GHC

cabal-testsuite/src/Test/Cabal/Prelude.hs

@@ -36,7 +36,6 @@ import Distribution.Simple.Configure
 import Distribution.Version
 import Distribution.Package
 import Distribution.Parsec (eitherParsec)
-import Distribution.Pretty (prettyShow)
 import Distribution.Types.UnqualComponentName
 import Distribution.Types.LocalBuildInfo
 import Distribution.PackageDescription
@@ -765,7 +764,7 @@ getScriptCacheDirectory script = do
 
 hasSharedLibraries  :: TestM Bool
 hasSharedLibraries = do
-    shared_libs_were_removed <- ghcVersionIs (>= mkVersion [7,8])
+    shared_libs_were_removed <- isGhcVersion ">= 7.8"
     return (not (buildOS == Windows && shared_libs_were_removed))
 
 hasProfiledLibraries :: TestM Bool
@@ -789,13 +788,23 @@ hasCabalShared = do
   env <- getTestEnv
   return (testHaveCabalShared env)
 
-ghcVersionIs :: WithCallStack ((Version -> Bool) -> TestM Bool)
-ghcVersionIs f = do
+isGhcVersion :: WithCallStack (String -> TestM Bool)
+isGhcVersion range = do
     ghc_program <- requireProgramM ghcProgram
-    case programVersion ghc_program of
-        Nothing -> error $ "ghcVersionIs: no ghc version for "
+    v <- case programVersion ghc_program of
+        Nothing -> error $ "isGhcVersion: no ghc version for "
                         ++ show (programLocation ghc_program)
-        Just v -> return (f v)
+        Just v -> return v
+    vr <- case eitherParsec range of
+        Left err -> fail err
+        Right vr -> return vr
+    return (v `withinRange` vr)
+
+skipUnlessGhcVersion :: String -> TestM ()
+skipUnlessGhcVersion range = skipUnless ("needs ghc " ++ range) =<< isGhcVersion range
+
+skipIfGhcVersion :: String -> TestM ()
+skipIfGhcVersion range = skipUnless ("incompatible with ghc " ++ range) =<< isGhcVersion range
 
 isWindows :: TestM Bool
 isWindows = return (buildOS == Windows)
@@ -809,12 +818,6 @@ isLinux = return (buildOS == Linux)
 skipIfWindows :: TestM ()
 skipIfWindows = skipIf "Windows" =<< isWindows
 
-skipUnlessGhcVersion :: String -> TestM ()
-skipUnlessGhcVersion str =
-    case eitherParsec str of
-        Right vr -> skipUnless ("needs ghc" ++ prettyShow vr) =<< ghcVersionIs (`withinRange` vr)
-        Left err -> fail err
-
 getOpenFilesLimit :: TestM (Maybe Integer)
 #ifdef mingw32_HOST_OS
 -- No MS-specified limit, was determined experimentally on Windows 10 Pro x64,
@@ -855,7 +858,7 @@ hasCabalForGhc = do
 -- You'll want to exclude them in that case.
 --
 hasNewBuildCompatBootCabal :: TestM Bool
-hasNewBuildCompatBootCabal = ghcVersionIs (>= mkVersion [7,9])
+hasNewBuildCompatBootCabal = isGhcVersion ">= 7.9"
 
 ------------------------------------------------------------------------
 -- * Broken tests
@@ -959,7 +962,7 @@ getIPID pn = do
 delay :: TestM ()
 delay = do
     env <- getTestEnv
-    is_old_ghc <- ghcVersionIs (< mkVersion [7,7])
+    is_old_ghc <- isGhcVersion "< 7.7"
     -- For old versions of GHC, we only had second-level precision,
     -- so we need to sleep a full second.  Newer versions use
     -- millisecond level precision, so we only have to wait