Fix retries for successful 500s (#570)

* fix: handle retries for failed successful http reqs * added http 429, specific 500s list, more diag Ref: LOG-18402 --------- Co-authored-by: Chris Nixon <[email protected]>
logdna · Oct 13, 2023 · f63cbe4 · f63cbe4
1 parent 5c44414
commit f63cbe4
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 6 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/bin/Cargo.toml b/bin/Cargo.toml
@@ -54,8 +54,8 @@ miniz_oxide = "0.5"
 rand = "0.8.5"
 shell-words = "1.0"
 async-channel = "1.8"
-
 rlimit = "0.10"
+rate-limit-macro = "1"
 
 [target.'cfg(any(windows))'.dependencies]
 winservice = { git = "https://github.com/dkhokhlov/winservice.git" }

diff --git a/bin/src/_main.rs b/bin/src/_main.rs
@@ -543,11 +543,11 @@ pub async fn _main(
                     error!("bad request, check configuration: {}", s);
                     shutdown_tx.lock().await.take().unwrap().send(()).unwrap();
                 } else {
-                    warn!("bad http request: {}", s);
+                    error!("bad http request: {}", s);
                 }
             }
             ClientError::Http(e) => {
-                warn!("failed sending http request: {}", e);
+                error!("failed sending http request: {}", e);
             }
             ClientError::Retry(r) => {
                 error!("failed to retry request: {}", r);
@@ -563,6 +563,11 @@ pub async fn _main(
             SendStatus::Retry(e) => {
                 warn!("failed sending http request, retrying: {}", e);
             }
+            SendStatus::RetryServerError(code, e) => {
+                rate_limit_macro::rate_limit!(rate = 1, interval = 1 * 60, {
+                    warn!("failed sending http request, retrying: {} {}", code, e);
+                });
+            }
             SendStatus::RetryTimeout => {
                 warn!("failed sending http request, retrying: request timed out!");
             }

diff --git a/common/http/src/client.rs b/common/http/src/client.rs
@@ -11,7 +11,7 @@ use crate::types::response::Response;
 
 use metrics::Metrics;
 use state::{FileOffsetFlushHandle, FileOffsetWriteHandle, OffsetMap};
-use tracing::{debug, error, warn};
+use tracing::{debug, error};
 
 /// Http(s) client used to send logs to the Ingest API
 pub struct Client {
@@ -25,6 +25,7 @@ pub struct Client {
 pub enum SendStatus {
     Sent,
     Retry(hyper::Error),
+    RetryServerError(hyper::StatusCode, String),
     RetryTimeout,
 }
 
@@ -94,14 +95,22 @@ impl Client {
             .send(self.limiter.get_slot(body).await.as_ref().clone())
             .await
         {
+            Ok(Response::Failed(body, s, r))
+                if [500, 501, 502, 503, 504, 507, 429].contains(&s.as_u16()) =>
+            {
+                Metrics::http().add_request_failure(start);
+                debug!("failed request, retrying: {} {}", s, r);
+                self.retry.retry(file_offsets, &body).await?;
+                Ok(SendStatus::RetryServerError(s, r))
+            }
             Ok(Response::Failed(_, s, r)) => {
                 Metrics::http().add_request_failure(start);
-                debug!("Failed request: {}", r);
+                debug!("failed request: {} {}", s, r);
                 Err(ClientError::BadRequest(s))
             }
             Err(HttpError::Send(body, e)) => {
                 Metrics::http().add_request_failure(start);
-                warn!("failed sending http request, retrying: {}", e);
+                debug!("failed sending http request, retrying: {}", e);
                 self.retry.retry(file_offsets, &body).await?;
                 Ok(SendStatus::Retry(e))
             }