Add more retries for 429s during teardown operations. (#8112)

Closes #7384.
googleapis · May 23, 2019 · a02e99e · a02e99e
1 parent 967242c
commit a02e99e
Showing 1 changed file with 9 additions and 8 deletions.
diff --git a/storage/tests/system.py b/storage/tests/system.py
@@ -45,6 +45,7 @@ def _bad_copy(bad_request):
 
 
 retry_429 = RetryErrors(exceptions.TooManyRequests, max_tries=6)
+retry_429_harder = RetryErrors(exceptions.TooManyRequests, max_tries=10)
 retry_429_503 = RetryErrors(
     [exceptions.TooManyRequests, exceptions.ServiceUnavailable], max_tries=6
 )
@@ -110,7 +111,7 @@ def setUp(self):
     def tearDown(self):
         for bucket_name in self.case_buckets_to_delete:
             bucket = Config.CLIENT.bucket(bucket_name)
-            retry_429(bucket.delete)()
+            retry_429_harder(bucket.delete)()
 
     def test_create_bucket(self):
         new_bucket_name = "a-new-bucket" + unique_resource_id("-")
@@ -299,7 +300,7 @@ def test_copy_existing_file_with_user_project(self):
             self.assertEqual(base_contents, copied_contents)
         finally:
             for blob in to_delete:
-                retry_429(blob.delete)()
+                retry_429_harder(blob.delete)()
 
     @unittest.skipUnless(USER_PROJECT, "USER_PROJECT not set in environment.")
     def test_bucket_get_blob_with_user_project(self):
@@ -1011,7 +1012,7 @@ def test_compose_with_user_project(self):
             composed = destination.download_as_string()
             self.assertEqual(composed, SOURCE_1 + SOURCE_2)
         finally:
-            retry_429(created.delete)(force=True)
+            retry_429_harder(created.delete)(force=True)
 
 
 class TestStorageRewrite(TestStorageFiles):
@@ -1085,7 +1086,7 @@ def test_rewrite_add_key_with_user_project(self):
 
             self.assertEqual(source.download_as_string(), dest.download_as_string())
         finally:
-            retry_429(created.delete)(force=True)
+            retry_429_harder(created.delete)(force=True)
 
     @unittest.skipUnless(USER_PROJECT, "USER_PROJECT not set in environment.")
     def test_rewrite_rotate_with_user_project(self):
@@ -1115,7 +1116,7 @@ def test_rewrite_rotate_with_user_project(self):
 
             self.assertEqual(dest.download_as_string(), source_data)
         finally:
-            retry_429(created.delete)(force=True)
+            retry_429_harder(created.delete)(force=True)
 
 
 class TestStorageUpdateStorageClass(TestStorageFiles):
@@ -1185,7 +1186,7 @@ def tearDown(self):
         with Config.CLIENT.batch():
             for bucket_name in self.case_buckets_to_delete:
                 bucket = Config.CLIENT.bucket(bucket_name)
-                retry_429(bucket.delete)()
+                retry_429_harder(bucket.delete)()
 
     @staticmethod
     def event_types():
@@ -1456,7 +1457,7 @@ def setUp(self):
     def tearDown(self):
         for bucket_name in self.case_buckets_to_delete:
             bucket = Config.CLIENT.bucket(bucket_name)
-            retry_429(bucket.delete)()
+            retry_429_harder(bucket.delete)()
 
     def test_bucket_w_retention_period(self):
         import datetime
@@ -1628,7 +1629,7 @@ def setUp(self):
     def tearDown(self):
         for bucket_name in self.case_buckets_to_delete:
             bucket = Config.CLIENT.bucket(bucket_name)
-            retry_429(bucket.delete)(force=True)
+            retry_429_harder(bucket.delete)(force=True)
 
     def test_new_bucket_w_bpo(self):
         new_bucket_name = "new-w-bpo" + unique_resource_id("-")