download_sysext: retry to download 20 times at max

If HTTP client fails to download, retry to download once in 1000 msec,
up to 20 times.

src/download.rs

@@ -4,11 +4,13 @@ use std::io;
 use std::fs::File;
 use std::path::Path;
 use log::info;
-use url::Url;
-
 use reqwest::StatusCode;
-
 use sha2::{Sha256, Digest};
+use tokio::time::{Duration, sleep};
+use url::Url;
+
+const MAX_DOWNLOAD_RETRY: u32 = 20;
+const RETRY_INTERVAL_MSEC: u64 = 1000;
 
 pub struct DownloadResult<W: std::io::Write> {
     pub hash: omaha::Hash<omaha::Sha256>,
@@ -63,59 +65,76 @@ where
     W: io::Write,
     Url: From<U>,
 {
-    let client_url = url.clone();
-
-    #[rustfmt::skip]
-    let mut res = client.get(url)
-        .send()
-        .await
-        .context(format!("client get and send({:?}) failed", client_url.as_str()))?;
-
-    // Redirect was already handled at this point, so there is no need to touch
-    // response or url again. Simply print info and continue.
-    if <U as Into<Url>>::into(client_url) != *res.url() {
-        info!("redirected to URL {:?}", res.url());
-    }
-
-    // Return immediately on download failure on the client side.
-    let status = res.status();
-
-    if !status.is_success() {
-        match status {
-            StatusCode::FORBIDDEN | StatusCode::NOT_FOUND => {
-                bail!("cannnot fetch remotely with status code {:?}", status);
-            }
-            _ => bail!("general failure with status code {:?}", status),
+    let mut try_index = 0;
+
+    // NOTE: a plain loop for retry like below is necessary to pass in data(W),
+    // which could be not clonable &mut File. Ideally the code below should be
+    // somehow split into a proper function passed into a retry loop, which is
+    // able to deal with an async block.
+    while try_index < MAX_DOWNLOAD_RETRY {
+        let client_url = url.clone();
+
+        #[rustfmt::skip]
+        let mut res = client.get(url.clone())
+            .send()
+            .await
+            .context(format!("client get and send({:?}) failed", client_url.as_str()))?;
+
+        // Redirect was already handled at this point, so there is no need to touch
+        // response or url again. Simply print info and continue.
+        if <U as Into<Url>>::into(client_url) != *res.url() {
+            info!("redirected to URL {:?}", res.url());
         }
-    }
 
-    let mut hasher = Sha256::new();
+        // Return immediately on download failure on the client side.
+        let status = res.status();
+
+        if !status.is_success() {
+            match status {
+                StatusCode::FORBIDDEN | StatusCode::NOT_FOUND => {
+                    info!("cannnot fetch remotely with status code {:?}, retrying...", status);
+                }
+                _ => {
+                    info!("general failure with status code {:?}, retrying...", status);
+                }
+            }
 
-    let mut bytes_read = 0usize;
-    let bytes_to_read = res.content_length().unwrap_or(u64::MAX) as usize;
+            sleep(Duration::from_millis(RETRY_INTERVAL_MSEC)).await;
+            try_index += 1;
+            continue;
+        }
 
-    while let Some(chunk) = res.chunk().await.context("failed to get response chunk")? {
-        bytes_read += chunk.len();
+        // response is success
+        let mut hasher = Sha256::new();
+        let mut bytes_read = 0usize;
+        let bytes_to_read = res.content_length().unwrap_or(u64::MAX) as usize;
+
+        while let Some(chunk) = res.chunk().await.context("failed to get response chunk")? {
+            bytes_read += chunk.len();
+
+            hasher.update(&chunk);
+            data.write_all(&chunk).context("failed to write_all chunk")?;
+
+            if print_progress {
+                print!(
+                    "\rread {}/{} ({:3}%)",
+                    bytes_read,
+                    bytes_to_read,
+                    ((bytes_read as f32 / bytes_to_read as f32) * 100.0f32).floor()
+                );
+                io::stdout().flush().context("failed to flush stdout")?;
+            }
+        }
 
-        hasher.update(&chunk);
-        data.write_all(&chunk).context("failed to write_all chunk")?;
+        data.flush().context("failed to flush data")?;
+        println!();
 
-        if print_progress {
-            print!(
-                "\rread {}/{} ({:3}%)",
-                bytes_read,
-                bytes_to_read,
-                ((bytes_read as f32 / bytes_to_read as f32) * 100.0f32).floor()
-            );
-            io::stdout().flush().context("failed to flush stdout")?;
-        }
+        return Ok(DownloadResult {
+            hash: omaha::Hash::from_bytes(hasher.finalize().into()),
+            data,
+        });
     }
 
-    data.flush().context("failed to flush data")?;
-    println!();
-
-    Ok(DownloadResult {
-        hash: omaha::Hash::from_bytes(hasher.finalize().into()),
-        data,
-    })
+    // No response success found, return error
+    bail!("unable to get a response succcess in {:?} attempts.", MAX_DOWNLOAD_RETRY);
 }