Added decorator for retry_request and handle err 429 in DBFS (#1491)

fsspec/implementations/dbfs.py

@@ -2,6 +2,8 @@
 import urllib
 
 import requests
+import requests.exceptions
+from requests.adapters import HTTPAdapter, Retry
 
 from fsspec import AbstractFileSystem
 from fsspec.spec import AbstractBufferedFile
@@ -42,13 +44,19 @@ def __init__(self, instance, token, **kwargs):
         """
         self.instance = instance
         self.token = token
-
         self.session = requests.Session()
+        self.retries = Retry(
+            total=10,
+            backoff_factor=0.05,
+            status_forcelist=[408, 429, 500, 502, 503, 504],
+        )
+
+        self.session.mount("https://", HTTPAdapter(max_retries=self.retries))
         self.session.headers.update({"Authorization": f"Bearer {self.token}"})
 
         super().__init__(**kwargs)
 
-    def ls(self, path, detail=True):
+    def ls(self, path, detail=True, **kwargs):
         """
         List the contents of the given path.
 
@@ -137,7 +145,7 @@ def mkdir(self, path, create_parents=True, **kwargs):
 
         self.mkdirs(path, **kwargs)
 
-    def rm(self, path, recursive=False):
+    def rm(self, path, recursive=False, **kwargs):
         """
         Remove the file or folder at the given absolute path.
 
@@ -166,7 +174,9 @@ def rm(self, path, recursive=False):
             raise e
         self.invalidate_cache(self._parent(path))
 
-    def mv(self, source_path, destination_path, recursive=False, maxdepth=None):
+    def mv(
+        self, source_path, destination_path, recursive=False, maxdepth=None, **kwargs
+    ):
         """
         Move a source to a destination path.
 

fsspec/implementations/tests/cassettes/test_dbfs_file_listing.yaml

@@ -13,22 +13,34 @@ interactions:
       Content-Type:
       - application/json
       User-Agent:
-      - python-requests/2.25.1
+      - python-requests/2.31.0
       authorization:
       - DUMMY
     method: GET
     uri: https://my_instance.com/api/2.0/dbfs/list
   response:
     body:
       string: !!binary |
-        H4sIAAAAAAAEA4zLMQ5AQBBG4bv89Qr1HsAFlCKyGDFByMxsQ9ydaHSy7cv3Toy8kMLXJ/ZgEzzy
-        8imVbUJwYG0HFniTSO61rfJB8MXlvmMIFjrhftZMSONimrwaqaXjdU+2UUn+cXPdAAAA//8DAHlY
-        NJf+AAAA
+        H4sIAAAAAAAEA6zSSwrCMBAG4LvMumLSpo3JAbyA4Eak1DbiYF8kExeW3t2qC6G46WM3/AwfzKOD
+        K5bGgT510GZ0Aw3b/ZAcqLEGAkCXFmhBk/Um+PSmDp8GNAugagq8Yp4RNnVKWA0pl0woxmMhGGN9
+        8DOPTemHhsngH8QtVIqMsovF/O4279IZWlG0xvlyMVjVNH1InoTD5kMpotHuqWrnaErGkeJyN9K8
+        M3YOxxmLEhGqEfdY4zG+yKwznvsXAAAA//8DAMDPrToDAwAA
     headers:
+      access-control-allow-headers:
+      - Authorization, X-Databricks-Azure-Workspace-Resource-Id, X-Databricks-Org-Id,
+        Content-Type
+      access-control-allow-origin:
+      - '*'
+      cache-control:
+      - no-cache, no-store, must-revalidate
       content-encoding:
       - gzip
       content-type:
       - application/json
+      expires:
+      - '0'
+      pragma:
+      - no-cache
       server:
       - databricks
       strict-transport-security:
@@ -56,22 +68,34 @@ interactions:
       Content-Type:
       - application/json
       User-Agent:
-      - python-requests/2.25.1
+      - python-requests/2.31.0
       authorization:
       - DUMMY
     method: GET
     uri: https://my_instance.com/api/2.0/dbfs/list
   response:
     body:
       string: !!binary |
-        H4sIAAAAAAAEA4zLMQ5AQBBG4bv89Qr1HsAFlCKyGDFByMxsQ9ydaHSy7cv3Toy8kMLXJ/ZgEzzy
-        8imVbUJwYG0HFniTSO61rfJB8MXlvmMIFjrhftZMSONimrwaqaXjdU+2UUn+cXPdAAAA//8DAHlY
-        NJf+AAAA
+        H4sIAAAAAAAEA6zSSwrCMBAG4LvMumLSpo3JAbyA4Eak1DbiYF8kExeW3t2qC6G46WM3/AwfzKOD
+        K5bGgT510GZ0Aw3b/ZAcqLEGAkCXFmhBk/Um+PSmDp8GNAugagq8Yp4RNnVKWA0pl0woxmMhGGN9
+        8DOPTemHhsngH8QtVIqMsovF/O4279IZWlG0xvlyMVjVNH1InoTD5kMpotHuqWrnaErGkeJyN9K8
+        M3YOxxmLEhGqEfdY4zG+yKwznvsXAAAA//8DAMDPrToDAwAA
     headers:
+      access-control-allow-headers:
+      - Authorization, X-Databricks-Azure-Workspace-Resource-Id, X-Databricks-Org-Id,
+        Content-Type
+      access-control-allow-origin:
+      - '*'
+      cache-control:
+      - no-cache, no-store, must-revalidate
       content-encoding:
       - gzip
       content-type:
       - application/json
+      expires:
+      - '0'
+      pragma:
+      - no-cache
       server:
       - databricks
       strict-transport-security: