Adding configs for retries, retrywait, timeout, and chunksize

src/fibad/download.py

@@ -73,7 +73,14 @@ def run(config):
     logger.info("Requesting cutouts")
     # pass the rects to the cutout downloader
     download_cutout_group(
-        rects=rects, cutout_dir=config.get("cutout_dir"), user=config["username"], password=config["password"]
+        rects=rects,
+        cutout_dir=config.get("cutout_dir"),
+        user=config["username"],
+        password=config["password"],
+        retrywait=config.get("retry_wait", 30),
+        retries=config.get("retries", 3),
+        timeout=config.get("timeout", 3600),
+        chunksize=config.get("chunk_size", 990),
     )
 
     logger.info("Done")
@@ -368,7 +375,7 @@ def save(self):
             missed.write(self.filepath, overwrite=True, **self.format_kwargs)
 
 
-def download_cutout_group(rects: list[dC.Rect], cutout_dir: Union[str, Path], user, password):
+def download_cutout_group(rects: list[dC.Rect], cutout_dir: Union[str, Path], user, password, **kwargs):
     """Download cutouts to the given directory
 
     Calls downloadCutout.download, so supports long lists of rects beyond the limits of the HSC web API
@@ -383,6 +390,8 @@ def download_cutout_group(rects: list[dC.Rect], cutout_dir: Union[str, Path], us
         Username for HSC's download service to use
     password : string
         Password for HSC's download service to use
+    **kwargs: dict
+        Additonal arguments for downloadCutout.download. See downloadCutout.download for details
     """
 
     with working_directory(Path(cutout_dir)):
@@ -398,5 +407,5 @@ def download_cutout_group(rects: list[dC.Rect], cutout_dir: Union[str, Path], us
                 request_hook=stats_hook,
                 failed_chunk_hook=failed_chunk_hook,
                 resume=True,
-                chunksize=10,
+                **kwargs,
             )

src/fibad/fibad_default_config.toml

@@ -30,6 +30,16 @@ cutout_dir = "../hscplay/cutouts/"
 offset = 0
 num_sources = 10
 
+# These control the downloader's HTTP requests and retries
+# `retry_wait` How long to wait before retrying a failed HTTP request in seconds. Default 30s
+retry_wait = 30
+# `retries` How many times to retry a failed HTTP request before moving on to the next one. Default 3 times
+retries = 3
+# `timepout` How long should we wait to get a full HTTP response from the server. Default 3600s (1hr)
+timeout = 3600
+# `chunksize` How many sky location rectangles should we request in a single request. Default is 990
+chunksize = 990
+
 [model]
 name = "ExampleCNN"