Perform exponential backoff with jitter when retrying requests

Cargo.toml

@@ -27,6 +27,7 @@ http = "1.1.0"
 log = "0.4.22"
 model = { path = "crates/model" }
 oauth2 = "4.4.2"
+rand = "0.9.0"
 rayon = "1.10.0"
 reqwest = { version = "0.11.23", default-features = false, features = [ "blocking", "json", "rustls-tls", ] }
 rusoto_core = { version = "0.48.0", default-features = false, features = [ "rustls", ] }

src/retry.rs

@@ -1,28 +1,54 @@
 use super::*;
 
 pub(crate) trait Retry {
-  fn retry(self, retries: usize) -> Result<reqwest::blocking::Response>;
+  fn retry(self, max_retries: usize) -> Result<reqwest::blocking::Response>;
 }
 
 impl Retry for RequestBuilder {
-  fn retry(self, retries: usize) -> Result<reqwest::blocking::Response> {
+  fn retry(self, max_retries: usize) -> Result<reqwest::blocking::Response> {
     let mut attempts = 0;
 
-    while attempts <= retries {
+    let (base_delay, max_delay) =
+      (Duration::from_millis(100), Duration::from_secs(30));
+
+    while attempts <= max_retries {
       match self
         .try_clone()
         .ok_or_else(|| anyhow!("Failed to clone request builder"))?
         .send()
       {
         Ok(response) => return Ok(response),
         Err(error) => {
-          error!("Request failed: {error}");
+          error!(
+            "Request failed (attempt {}/{}): {}",
+            attempts + 1,
+            max_retries + 1,
+            error
+          );
+
+          if attempts == max_retries {
+            break;
+          }
+
+          let (exp_delay, jitter) = (
+            base_delay * (2_u32.pow(attempts as u32)),
+            Duration::from_millis(rand::random::<u64>() % 100),
+          );
+
+          let delay = std::cmp::min(exp_delay + jitter, max_delay);
+
+          warn!("Retrying in {:?}...", delay);
+
+          thread::sleep(delay);
+
           attempts += 1;
-          thread::sleep(Duration::from_secs(1));
         }
       }
     }
 
-    Err(Error(anyhow!("Request timed out")))
+    Err(Error(anyhow!(
+      "Request failed after {} attempts",
+      max_retries + 1
+    )))
   }
 }