Adds watchtower-plugin autoretry tests

watchtower-plugin/src/retrier.rs

@@ -507,7 +507,8 @@ mod tests {
     use crate::test_utils::get_dummy_add_appointment_response;
 
     const MAX_ELAPSED_TIME: u16 = 2;
-    const AUTO_RETRY_TIME: u16 = 5;
+    const LONG_AUTO_RETRY_DELAY: u16 = 60;
+    const SHORT_AUTO_RETRY_DELAY: u16 = 5;
     const MAX_INTERVAL_TIME: u16 = 1;
 
     impl Retrier {
@@ -571,7 +572,7 @@ mod tests {
                 wt_client_clone,
                 rx,
                 MAX_ELAPSED_TIME,
-                AUTO_RETRY_TIME,
+                LONG_AUTO_RETRY_DELAY,
                 MAX_INTERVAL_TIME,
             )
             .manage_retry()
@@ -613,7 +614,7 @@ mod tests {
         ));
 
         // Add a tower with pending appointments
-        let (_, tower_pk) = cryptography::get_random_keypair();
+        let (tower_sk, tower_pk) = cryptography::get_random_keypair();
         let tower_id = TowerId(tower_pk);
         let receipt = get_random_registration_receipt();
         wt_client
@@ -633,17 +634,14 @@ mod tests {
         let wt_client_clone = wt_client.clone();
 
         let max_elapsed_time = MAX_ELAPSED_TIME + 1;
-        let task = tokio::spawn(async move {
-            RetryManager::new(
-                wt_client_clone,
-                rx,
-                MAX_ELAPSED_TIME,
-                AUTO_RETRY_TIME,
-                MAX_INTERVAL_TIME,
-            )
-            .manage_retry()
-            .await
-        });
+        let mut retry_manager = RetryManager::new(
+            wt_client_clone,
+            rx,
+            MAX_ELAPSED_TIME,
+            SHORT_AUTO_RETRY_DELAY,
+            MAX_INTERVAL_TIME,
+        );
+        let task = tokio::spawn(async move { retry_manager.manage_retry().await });
         tx.send((tower_id, HashSet::from([appointment.locator])))
             .unwrap();
 
@@ -665,6 +663,56 @@ mod tests {
             .unwrap()
             .is_unreachable());
 
+        // Add a proper server and check that the auto-retry works
+        // Prepare the mock response
+        let server = MockServer::start();
+        let mut add_appointment_receipt = AppointmentReceipt::new(
+            cryptography::sign(&appointment.to_vec(), &wt_client.lock().unwrap().user_sk).unwrap(),
+            42,
+        );
+        add_appointment_receipt.sign(&tower_sk);
+        let add_appointment_response =
+            get_dummy_add_appointment_response(appointment.locator, &add_appointment_receipt);
+        let api_mock = server.mock(|when, then| {
+            when.method(POST).path("/add_appointment");
+            then.status(200)
+                .header("content-type", "application/json")
+                .json_body(json!(add_appointment_response));
+        });
+
+        // Update the tower details
+        wt_client
+            .lock()
+            .unwrap()
+            .add_update_tower(
+                tower_id,
+                &server.base_url(),
+                &get_registration_receipt_from_previous(&receipt),
+            )
+            .unwrap();
+
+        // Wait and check. We wait twice the short retry delay because it can be the case that the first auto retry
+        // is performed while we are patching the mock.
+        tokio::time::sleep(Duration::from_secs((SHORT_AUTO_RETRY_DELAY * 2) as u64)).await;
+        assert_eq!(
+            wt_client
+                .lock()
+                .unwrap()
+                .get_tower_status(&tower_id)
+                .unwrap(),
+            TowerStatus::Reachable
+        );
+        assert!(!wt_client
+            .lock()
+            .unwrap()
+            .towers
+            .get(&tower_id)
+            .unwrap()
+            .pending_appointments
+            .contains(&appointment.locator));
+
+        api_mock.assert();
+
         task.abort();
     }
 
@@ -712,7 +760,7 @@ mod tests {
                 wt_client_clone,
                 rx,
                 MAX_ELAPSED_TIME,
-                AUTO_RETRY_TIME,
+                LONG_AUTO_RETRY_DELAY,
                 MAX_INTERVAL_TIME,
             )
             .manage_retry()
@@ -801,7 +849,7 @@ mod tests {
                 wt_client_clone,
                 rx,
                 MAX_ELAPSED_TIME,
-                AUTO_RETRY_TIME,
+                LONG_AUTO_RETRY_DELAY,
                 MAX_INTERVAL_TIME,
             )
             .manage_retry()
@@ -852,7 +900,7 @@ mod tests {
                 wt_client_clone,
                 rx,
                 MAX_ELAPSED_TIME,
-                AUTO_RETRY_TIME,
+                LONG_AUTO_RETRY_DELAY,
                 MAX_INTERVAL_TIME,
             )
             .manage_retry()
@@ -938,7 +986,7 @@ mod tests {
                 wt_client_clone,
                 rx,
                 MAX_ELAPSED_TIME,
-                AUTO_RETRY_TIME,
+                LONG_AUTO_RETRY_DELAY,
                 MAX_INTERVAL_TIME,
             )
             .manage_retry()

watchtower-plugin/tests/test.py

@@ -111,10 +111,10 @@ def test_unreachable_watchtower(node_factory, bitcoind, teosd):
     assert l2.rpc.gettowerinfo(tower_id)["status"] == "reachable"
 
 
-def test_retry_watchtower(node_factory, bitcoind, teosd):
+def test_auto_retry_watchtower(node_factory, bitcoind, teosd):
     # The plugin is set to give up on retrying straight-away so we can test this fast.
     l1, l2 = node_factory.line_graph(
-        2, opts=[{}, {"plugin": WT_PLUGIN, "allow_broken_log": True, "watchtower-max-retry-time": 0}]
+        2, opts=[{}, {"plugin": WT_PLUGIN, "allow_broken_log": True, "watchtower-max-retry-time": 1, "watchtower-auto-retry-delay": 1}]
     )
 
     # We need to register l2 with the tower
@@ -128,18 +128,47 @@ def test_retry_watchtower(node_factory, bitcoind, teosd):
     l1.rpc.pay(l2.rpc.invoice(25000000, "lbl1", "desc1")["bolt11"])
 
     # Wait until the tower has been flagged as unreachable
-    l2.daemon.wait_for_log(f"Setting {tower_id} as unreachable")
+    l2.daemon.wait_for_log(f"Starting to idle")
     assert l2.rpc.gettowerinfo(tower_id)["status"] == "unreachable"
     assert l2.rpc.gettowerinfo(tower_id)["pending_appointments"]
 
     # Start the tower and retry it
     teosd.start()
 
-    l2.rpc.retrytower(tower_id)
+    l2.daemon.wait_for_log(f"Finished idling. Flagging {tower_id} for retry")
     l2.daemon.wait_for_log(f"Retry strategy succeeded for {tower_id}")
     assert l2.rpc.gettowerinfo(tower_id)["status"] == "reachable"
 
 
+def test_manually_retry_watchtower(node_factory, bitcoind, teosd):
+    # The plugin is set to give up on retrying straight-away so we can test this fast.
+    l1, l2 = node_factory.line_graph(
+        2, opts=[{}, {"plugin": WT_PLUGIN, "allow_broken_log": True, "watchtower-max-retry-time": 0}]
+    )
+
+    # We need to register l2 with the tower
+    tower_id = teosd.cli.gettowerinfo()["tower_id"]
+    l2.rpc.registertower(tower_id)
+
+    # Stop the tower
+    teosd.stop()
+
+    # Make a new payment with an unreachable tower
+    l1.rpc.pay(l2.rpc.invoice(25000000, "lbl1", "desc1")["bolt11"])
+
+    # Wait until the tower has been flagged as unreachable
+    l2.daemon.wait_for_log(f"Starting to idle")
+    assert l2.rpc.gettowerinfo(tower_id)["status"] == "unreachable"
+    assert l2.rpc.gettowerinfo(tower_id)["pending_appointments"]
+
+    # Start the tower and retry it
+    teosd.start()
+
+    # Manual retry
+    l2.rpc.retrytower(tower_id)
+    l2.daemon.wait_for_log(f"Manually finished idling. Flagging {tower_id} for retry")
+
+
 def test_misbehaving_watchtower(node_factory, bitcoind, teosd, directory):
     l1, l2 = node_factory.line_graph(2, opts=[{}, {"plugin": WT_PLUGIN, "allow_broken_log": True}])