From d0fc16ed506be489fa6ddc486ef7a419bb086c0d Mon Sep 17 00:00:00 2001 From: Johannes Lund Date: Thu, 19 Nov 2020 16:30:01 +0100 Subject: [PATCH 1/3] Add flakyBecauseOf hspec helper like pendingWith, but disabled by setting export RUN_FLAKY_TESTS=1 This can be done before release. --- .../Integration/Scenario/API/Shelley/StakePools.hs | 2 +- lib/test-utils/src/Test/Hspec/Extra.hs | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/core-integration/src/Test/Integration/Scenario/API/Shelley/StakePools.hs b/lib/core-integration/src/Test/Integration/Scenario/API/Shelley/StakePools.hs index 4ba301f4298..265b4b0afd7 100644 --- a/lib/core-integration/src/Test/Integration/Scenario/API/Shelley/StakePools.hs +++ b/lib/core-integration/src/Test/Integration/Scenario/API/Shelley/StakePools.hs @@ -85,7 +85,7 @@ import Test.Hspec import Test.Hspec.Expectations.Lifted ( shouldBe, shouldSatisfy ) import Test.Hspec.Extra - ( it ) + ( flakyBecauseOf, it ) import Test.Integration.Framework.Context ( Context (..), PoolGarbageCollectionEvent (..) ) import Test.Integration.Framework.DSL diff --git a/lib/test-utils/src/Test/Hspec/Extra.hs b/lib/test-utils/src/Test/Hspec/Extra.hs index c4dabcbfd8a..080566c8e25 100644 --- a/lib/test-utils/src/Test/Hspec/Extra.hs +++ b/lib/test-utils/src/Test/Hspec/Extra.hs @@ -14,6 +14,7 @@ module Test.Hspec.Extra ( aroundAll , it , itWithCustomTimeout + , flakyBecauseOf ) where import Prelude @@ -26,8 +27,11 @@ import Control.Concurrent.MVar ( MVar, newEmptyMVar, putMVar, takeMVar ) import Control.Exception ( SomeException, catch, throwIO ) +import System.Environment + ( lookupEnv ) import Test.Hspec ( ActionWith + , Expectation , HasCallStack , Spec , SpecWith @@ -35,6 +39,7 @@ import Test.Hspec , beforeAll , beforeWith , expectationFailure + , pendingWith , specify ) @@ -143,3 +148,11 @@ await = takeMVar -- | Some helper to help readability on the thread synchronization above. unlock :: MVar () -> IO () unlock = flip putMVar () + +-- | Mark a test pending because of flakiness, with given reason. Unless the +-- RUN_FLAKY_TESTS environment variable is set. +flakyBecauseOf :: String -> Expectation +flakyBecauseOf ticketOrReason = + lookupEnv "RUN_FLAKY_TESTS" >>= \case + Just _ -> return () + Nothing -> pendingWith $ "Flaky: " <> ticketOrReason From 64ef342726d81fc7d42510c419ac89b228015a40 Mon Sep 17 00:00:00 2001 From: Johannes Lund Date: Thu, 19 Nov 2020 16:30:06 +0100 Subject: [PATCH 2/3] Mark TRANS_TTL_{01,02} and STAKE_POOLS_JOIN_05 flaky --- .../Integration/Scenario/API/Shelley/StakePools.hs | 1 + .../Integration/Scenario/API/Shelley/Transactions.hs | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/core-integration/src/Test/Integration/Scenario/API/Shelley/StakePools.hs b/lib/core-integration/src/Test/Integration/Scenario/API/Shelley/StakePools.hs index 265b4b0afd7..aa1ce773bcf 100644 --- a/lib/core-integration/src/Test/Integration/Scenario/API/Shelley/StakePools.hs +++ b/lib/core-integration/src/Test/Integration/Scenario/API/Shelley/StakePools.hs @@ -581,6 +581,7 @@ spec = describe "SHELLEY_STAKE_POOLS" $ do it "STAKE_POOLS_JOIN_05 - \ \Can join when stake key already exists" $ \ctx -> runResourceT $ do + liftIO $ flakyBecauseOf "#2230" let walletWithPreRegKey = [ "over", "decorate", "flock", "badge", "beauty" , "stamp" , "chest", "owner", "excess", "omit" diff --git a/lib/core-integration/src/Test/Integration/Scenario/API/Shelley/Transactions.hs b/lib/core-integration/src/Test/Integration/Scenario/API/Shelley/Transactions.hs index a1f50486899..f808cef8aea 100644 --- a/lib/core-integration/src/Test/Integration/Scenario/API/Shelley/Transactions.hs +++ b/lib/core-integration/src/Test/Integration/Scenario/API/Shelley/Transactions.hs @@ -88,11 +88,11 @@ import Network.HTTP.Types.Method import Numeric.Natural ( Natural ) import Test.Hspec - ( SpecWith, describe, pendingWith ) + ( SpecWith, describe ) import Test.Hspec.Expectations.Lifted ( expectationFailure, shouldBe, shouldNotBe, shouldSatisfy ) import Test.Hspec.Extra - ( it ) + ( flakyBecauseOf, it ) import Test.Integration.Framework.DSL ( Context , Headers (..) @@ -607,6 +607,7 @@ spec = describe "SHELLEY_TRANSACTIONS" $ do let slotDiff a b = if a > b then a - b else b - a it "TRANS_TTL_01 - Pending transaction expiry" $ \ctx -> runResourceT $ do + liftIO $ flakyBecauseOf "#2295" (wa, wb) <- (,) <$> fixtureWallet ctx <*> emptyWallet ctx let amt = minUTxOValue :: Natural @@ -640,6 +641,7 @@ spec = describe "SHELLEY_TRANSACTIONS" $ do & counterexample ("actual expiry: " <> show txActualExp) it "TRANS_TTL_02 - Custom transaction expiry" $ \ctx -> runResourceT $ do + liftIO $ flakyBecauseOf "#2295" (wa, wb) <- (,) <$> fixtureWallet ctx <*> emptyWallet ctx let amt = minUTxOValue :: Natural let testTTL = 42 :: NominalDiffTime @@ -674,7 +676,7 @@ spec = describe "SHELLEY_TRANSACTIONS" $ do & counterexample ("actual expiry: " <> show txActualExp) it "TRANS_TTL_03 - Expired transactions" $ \ctx -> runResourceT $ do - liftIO $ pendingWith "#1840 this is flaky -- need a better approach" + liftIO $ flakyBecauseOf "#1840 -- need a better approach" (wa, wb) <- (,) <$> fixtureWallet ctx <*> emptyWallet ctx let amt = minUTxOValue :: Natural @@ -2219,7 +2221,7 @@ spec = describe "SHELLEY_TRANSACTIONS" $ do txDeleteFromDifferentWalletTest emptyRandomWallet "byron-wallets" it "TRANS_TTL_DELETE_01 - Shelley: can remove expired tx" $ \ctx -> runResourceT $ do - liftIO $ pendingWith "#1840 this is flaky -- need a better approach" + liftIO $ flakyBecauseOf "#1840 -- need a better approach" (wa, wb) <- (,) <$> fixtureWallet ctx <*> emptyWallet ctx let amt = minUTxOValue :: Natural From 16da74a7acad7c11db7727d634c45c9d6ef7f8e1 Mon Sep 17 00:00:00 2001 From: Johannes Lund Date: Thu, 19 Nov 2020 16:42:19 +0100 Subject: [PATCH 3/3] Add FlakyTests.md manual test --- test/manual/cardano-node/FlakyTests.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 test/manual/cardano-node/FlakyTests.md diff --git a/test/manual/cardano-node/FlakyTests.md b/test/manual/cardano-node/FlakyTests.md new file mode 100644 index 00000000000..710c18ce920 --- /dev/null +++ b/test/manual/cardano-node/FlakyTests.md @@ -0,0 +1,15 @@ +# Run Flaky Tests + +Some tests might have been disabled in CI with the `flakyBecauseOf` helper. + +Run them locally using + +```bash +RUN_FLAKY_TESTS=1 stack test cardano-wallet:integration +``` + +or on Windows: +```bash +set RUN_FLAKY_TESTS=1 +cardano-wallet-test-integration.exe +```