Adjust log levels and log information (#7)

helius-labs · Apr 3, 2023 · b15e991 · b15e991
1 parent c8f1634
commit b15e991
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 26 deletions.
diff --git a/README.md b/README.md
@@ -234,6 +234,7 @@ time ingester.bgtask.proc_time
 count ingester.bgtask.success
 count ingester.bgtask.error
 count ingester.bgtask.network_error
+count ingester.bgtask.unrecoverable_error
 time ingester.bgtask.bus_time
 count ingester.bgtask.identical
 

diff --git a/nft_ingester/src/backfiller.rs b/nft_ingester/src/backfiller.rs
@@ -493,8 +493,7 @@ impl<'a, T: Messenger> Backfiller<'a, T> {
         if missing_trees.len() > 0 {
             info!("Number of Missing local trees: {}", missing_trees.len());
         } else {
-            debug!("No missing trees"
-            )
+            debug!("No missing trees");
         }
         Ok(missing_trees)
     }

diff --git a/nft_ingester/src/error/mod.rs b/nft_ingester/src/error/mod.rs
@@ -44,8 +44,8 @@ pub enum IngesterError {
     UnknownTaskType(String),
     #[error("BG Task Manager Not Started")]
     TaskManagerNotStarted,
-    #[error("Unrecoverable task error")]
-    UnrecoverableTaskError,
+    #[error("Unrecoverable task error: {0}")]
+    UnrecoverableTaskError(String),
     #[error("Cache Storage Write Error {0}")]
     CacheStorageWriteError(String),
 }

diff --git a/nft_ingester/src/metrics.rs b/nft_ingester/src/metrics.rs
@@ -2,7 +2,7 @@ use std::net::UdpSocket;
 
 use cadence::{BufferedUdpMetricSink, QueuingMetricSink, StatsdClient};
 use cadence_macros::{is_global_default_set, set_global_default, statsd_count, statsd_time};
-use log::error;
+use log::{error, warn};
 use tokio::time::Instant;
 
 use crate::{
@@ -75,14 +75,14 @@ pub fn capture_result(
             metric! {
                 statsd_count!("ingester.ingest_error", 1, label.0 => &label.1, "stream" => stream, "error" => "de");
             }
-            error!("{}", e);
+            warn!("{}", e);
             ret_id = Some(id);
         }
         Err(IngesterError::ParsingError(e)) => {
             metric! {
                 statsd_count!("ingester.ingest_error", 1, label.0 => &label.1, "stream" => stream, "error" => "parse");
             }
-            error!("{}", e);
+            warn!("{}", e);
             ret_id = Some(id);
         }
         Err(err) => {

diff --git a/nft_ingester/src/tasks/common/mod.rs b/nft_ingester/src/tasks/common/mod.rs
@@ -86,7 +86,7 @@ impl BgTask for DownloadMetadataTask {
         let download_metadata: DownloadMetadata = serde_json::from_value(data)?;
         let meta_url = Url::parse(&download_metadata.uri);
         let body = match meta_url {
-            Ok(_) => DownloadMetadataTask::request_metadata(download_metadata.uri).await?,
+            Ok(_) => DownloadMetadataTask::request_metadata(download_metadata.uri.clone()).await?,
             _ => serde_json::Value::String("Invalid Uri".to_string()), //TODO -> enumize this.
         };
         let model = asset_data::ActiveModel {
@@ -111,7 +111,10 @@ impl BgTask for DownloadMetadataTask {
                 ))
             })?;
         if meta_url.is_err() {
-            return Err(IngesterError::UnrecoverableTaskError);
+            return Err(IngesterError::UnrecoverableTaskError(format!(
+                "Failed to parse URI: {}",
+                download_metadata.uri
+            )));
         }
         Ok(())
     }

diff --git a/nft_ingester/src/tasks/mod.rs b/nft_ingester/src/tasks/mod.rs
@@ -112,29 +112,43 @@ impl TaskManager {
                 task.locked_by = Set(None);
             }
             Err(e) => {
-                if e == IngesterError::UnrecoverableTaskError {
-                    task.attempts = Set(task_def.max_attempts() + 1);
-                    task.locked_by = Set(Some("permanent failure".to_string()));
-                // todo add new task status
-                } else {
+                let err_msg = e.to_string();
+                match e {
+                    IngesterError::UnrecoverableTaskError(_) => {
+                        task.attempts = Set(task_def.max_attempts() + 1);
+                        task.locked_by = Set(Some("permanent failure".to_string()));
+                    },
+                    _ => {
                     task.locked_by = Set(None);
+                    }
                 }
                 task.status = Set(TaskStatus::Failed);
-                task.errors = Set(Some(e.to_string()));
+                task.errors = Set(Some(err_msg));
                 task.locked_until = Set(None);
 
-                if e == IngesterError::BatchInitNetworkingError {
-                    // Network errors are common for off-chain JSONs.
-                    // Logging these as errors is far too noisy.
-                    metric! {
-                        statsd_count!("ingester.bgtask.network_error", 1, "type" => task_name);
-                    }
-                    warn!("Task failed due to network error: {}",  e);
-                } else {
-                    metric! {
-                        statsd_count!("ingester.bgtask.error", 1, "type" => task_name);
+                match e {
+                    IngesterError::BatchInitNetworkingError => {
+                        // Network errors are common for off-chain JSONs.
+                        // Logging these as errors is far too noisy.
+                        metric! {
+                            statsd_count!("ingester.bgtask.network_error", 1, "type" => task_name);
+                        }
+                        warn!("Task failed due to network error: {}",  e);
+                    },
+                    IngesterError::UnrecoverableTaskError(_) => {
+                        // Unrecoverable errors are always going to be off-chain parsing failures at the moment.
+                        // We can't do anything about malformed JSONs.
+                        metric! {
+                            statsd_count!("ingester.bgtask.unrecoverable_error", 1, "type" => task_name);
+                        }
+                        warn!("{}",  e);
+                    },
+                    _ => {
+                        metric! {
+                            statsd_count!("ingester.bgtask.error", 1, "type" => task_name);
+                        }
+                        error!("Task Run Error: {}",  e);
                     }
-                    error!("Task Run Error: {}",  e);
                 }
             }
         }