fix(telemetry): retry telemetry ut 3 times to mitigate failures on CI (…

…#19983) Signed-off-by: MrCroxx <[email protected]>
risingwavelabs · Jan 2, 2025 · 104c4bf · 104c4bf
1 parent 5cb0d17
commit 104c4bf
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 6 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/src/compute/Cargo.toml b/src/compute/Cargo.toml
@@ -62,6 +62,7 @@ futures-async-stream = { workspace = true }
 rand = { workspace = true }
 risingwave_hummock_sdk = { workspace = true }
 tempfile = "3"
+tokio-retry = "0.3"
 
 [lints]
 workspace = true
diff --git a/src/compute/src/telemetry.rs b/src/compute/src/telemetry.rs
@@ -104,7 +104,13 @@ mod test {
         let pb_report = report.to_pb_bytes();
         let url =
             (TELEMETRY_REPORT_URL.to_owned() + "/" + TELEMETRY_COMPUTE_REPORT_TYPE).to_owned();
-        let post_res = post_telemetry_report_pb(&url, pb_report).await;
-        assert!(post_res.is_ok());
+
+        // Retry 3 times to mitigate occasional failures on CI.
+        tokio_retry::Retry::spawn(
+            tokio_retry::strategy::ExponentialBackoff::from_millis(10).take(3),
+            || post_telemetry_report_pb(&url, pb_report.clone()),
+        )
+        .await
+        .unwrap();
     }
 }
diff --git a/src/meta/src/telemetry.rs b/src/meta/src/telemetry.rs
@@ -282,7 +282,13 @@ mod test {
 
         let pb_bytes = report.to_pb_bytes();
         let url = (TELEMETRY_REPORT_URL.to_owned() + "/" + TELEMETRY_META_REPORT_TYPE).to_owned();
-        let result = post_telemetry_report_pb(&url, pb_bytes).await;
-        assert!(result.is_ok());
+
+        // Retry 3 times to mitigate occasional failures on CI.
+        tokio_retry::Retry::spawn(
+            tokio_retry::strategy::ExponentialBackoff::from_millis(10).take(3),
+            || post_telemetry_report_pb(&url, pb_bytes.clone()),
+        )
+        .await
+        .unwrap();
     }
 }
diff --git a/src/storage/compactor/Cargo.toml b/src/storage/compactor/Cargo.toml
@@ -34,6 +34,9 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [
 tonic = { workspace = true }
 tracing = "0.1"
 
+[dev-dependencies]
+tokio-retry = "0.3"
+
 [target.'cfg(not(madsim))'.dependencies]
 workspace-hack = { path = "../../workspace-hack" }
 

diff --git a/src/storage/compactor/src/telemetry.rs b/src/storage/compactor/src/telemetry.rs
@@ -126,7 +126,13 @@ mod test {
         let pb_report = report.to_pb_bytes();
         let url =
             (TELEMETRY_REPORT_URL.to_owned() + "/" + TELEMETRY_COMPACTOR_REPORT_TYPE).to_owned();
-        let post_res = post_telemetry_report_pb(&url, pb_report).await;
-        assert!(post_res.is_ok());
+
+        // Retry 3 times to mitigate occasional failures on CI.
+        tokio_retry::Retry::spawn(
+            tokio_retry::strategy::ExponentialBackoff::from_millis(10).take(3),
+            || post_telemetry_report_pb(&url, pb_report.clone()),
+        )
+        .await
+        .unwrap();
     }
 }