retry on http status of 429 (#215)

src/IC/Test/Agent.hs

@@ -423,7 +423,10 @@ postCBOR' ep path gr = do
       , requestBody = RequestBodyLBS $ BS.toLazyByteString $ encode gr
       , requestHeaders = [(hContentType, "application/cbor")]
       }
-    httpLbs request agentManager
+    waitFor $ do
+      res <- httpLbs request agentManager
+      if responseStatus res == tooManyRequests429 then return Nothing
+      else return $ Just res
 
 -- | postCBOR with url based on effective canister id
 postCBOR :: (HasCallStack, HasAgentConfig) => String -> GenR -> IO (Response BS.ByteString)
@@ -437,14 +440,16 @@ postCallCBOR cid      = (\r -> sync_height cid >> postCBOR ("/api/v2/canister/"
 postQueryCBOR cid     = (\r -> sync_height cid >> postCBOR ("/api/v2/canister/" ++ textual cid ++ "/query") r)
 postReadStateCBOR cid = (\r -> sync_height cid >> postReadStateCBOR' endPoint cid r)
 
-waitFor :: HasAgentConfig => IO Bool -> IO ()
+waitFor :: HasAgentConfig => IO (Maybe a) -> IO a
 waitFor act = do
     result <- timeout (tc_timeout agentConfig * (10::Int) ^ (6::Int)) doActUntil
-    when (result == Nothing) $ assertFailure "Polling timed out"
+    case result of Nothing -> assertFailure "Polling timed out"
+                   Just r -> return r
   where
     doActUntil = do
-      stop <- act
-      unless stop (threadDelay 1000 *> doActUntil)
+      res <- act
+      case res of Nothing -> (threadDelay 1000 *> doActUntil)
+                  Just r -> return r
 
 sync_height :: HasAgentConfig => Blob -> IO [()]
 sync_height cid = forM subnets $ \sub -> do
@@ -455,7 +460,7 @@ sync_height cid = forM subnets $ \sub -> do
     unless (length (nub hs) <= 1) $
       waitFor $ do
         hs <- get_heights (tc_node_addresses sub)
-        return $ h <= minimum hs
+        if h <= minimum hs then return (Just ()) else return Nothing
   where
     get_heights ns = mapM (\n -> do
       Right cert <- getStateCert'' n defaultUser cid [["time"]]

src/IC/Test/Spec/CanisterVersion.hs

@@ -35,7 +35,7 @@ canister_version_tests ecid =
     let simpleTestCase name ecid act = testCase name $ install ecid no_heartbeat >>= act in
     let get_global cid = query cid $ replyData getGlobal in
     let blob = toLazyByteString . word64LE . fromIntegral in
-    let wait_for_global cid n = waitFor $ (blob n ==) <$> get_global cid in
+    let wait_for_global cid n = waitFor $ (\b -> return $ if b then Just () else Nothing) <$> (blob n ==) <$> get_global cid in
     [ simpleTestCase "in query" ecid $ \cid -> do
       ctr <- query cid (replyData canister_version) >>= asWord64
       ctr @?= 1

src/IC/Test/Spec/Timer.hs

@@ -43,7 +43,7 @@ canister_timer_tests ecid =
     let get_far_far_future_time = floor . (* 1e9) <$> (+) 1000000 <$> getPOSIXTime in
     let set_timer cid time = call cid (replyData $ i64tob $ apiGlobalTimerSet $ int64 time) in
     let blob = toLazyByteString . word64LE . fromIntegral in
-    let wait_for_timer cid n = waitFor $ (blob n ==) <$> get_stable cid in
+    let wait_for_timer cid n = waitFor $ (\b -> return $ if b then Just () else Nothing) <$> (blob n ==) <$> get_stable cid in
     [ testCase "in update" $ do
       cid <- install_canister_with_global_timer (2::Int)
       _ <- reset_stable cid
@@ -95,7 +95,8 @@ canister_timer_tests ecid =
       _ <- ic_stop_canister ic00 cid
       waitFor $ do
         cs <- ic_canister_status ic00 cid
-        return $ cs .! #status == enum #stopped
+        if cs .! #status == enum #stopped then return $ Just ()
+        else return Nothing
       _ <- ic_install ic00 (enum #upgrade) cid universal_wasm (run $ on_timer_prog (2::Int) >>> set_timer_prog past_time)
       _ <- ic_start_canister ic00 cid
       wait_for_timer cid 2