[forge] add metrics for indexer processor latency & tuning

aptos-labs · Oct 10, 2024 · cc13744 · cc13744
1 parent 85bba15
commit cc13744
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 3 deletions.
diff --git a/.github/workflows/forge-unstable.yaml b/.github/workflows/forge-unstable.yaml
@@ -128,6 +128,7 @@ jobs:
     secrets: inherit
     with:
       IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
+      FORGE_NAMESPACE: forge-indexer-test-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
       COMMENT_HEADER: forge-indexer
       FORGE_TEST_SUITE: indexer_test
       FORGE_ENABLE_INDEXER: true
diff --git a/testsuite/forge-cli/src/main.rs b/testsuite/forge-cli/src/main.rs
@@ -2400,7 +2400,7 @@ fn indexer_test() -> ForgeConfig {
         (
             TransactionWorkload::new(TransactionTypeArg::TokenV2AmbassadorMint, 20000)
                 .with_unique_senders(),
-            (3200, 0.5, 0.5, 0.5),
+            (2000, 0.5, 0.5, 0.5),
         ),
         (
             TransactionWorkload::new(TransactionTypeArg::PublishPackage, 200)

diff --git a/testsuite/forge/src/interface/prometheus_metrics.rs b/testsuite/forge/src/interface/prometheus_metrics.rs
@@ -115,7 +115,7 @@ pub enum LatencyBreakdownSlice {
     IndexerFullnodeProcessedBatch,
     IndexerCacheWorkerProcessedBatch,
     IndexerDataServiceAllChunksSent,
-    // TODO: add processor insertion into DB latency
+    IndexerProcessorLatency,
 }
 
 #[derive(Clone, Debug)]
@@ -236,6 +236,11 @@ pub async fn fetch_latency_breakdown(
         let indexer_data_service_all_chunks_sent_query =
             r#"max(indexer_grpc_duration_in_secs{step="4", service_type="data_service"})"#;
 
+        // These are the non-SDK core indexer processors
+        let indexer_processor_latency_query =
+            r#"max by (processor_name) indexer_processor_data_processed_latency_in_secs"#;
+        // These are the SDK processor latencies TODO
+
         let indexer_fullnode_processed_batch_samples = swarm
             .query_range_metrics(
                 indexer_fullnode_processed_batch_query,
@@ -263,6 +268,15 @@ pub async fn fetch_latency_breakdown(
             )
             .await?;
 
+        let indexer_processor_latency_samples = swarm
+            .query_range_metrics(
+                indexer_processor_latency_query,
+                start_time as i64,
+                end_time as i64,
+                None,
+            )
+            .await?;
+
         samples.insert(
             LatencyBreakdownSlice::IndexerFullnodeProcessedBatch,
             MetricSamples::new(indexer_fullnode_processed_batch_samples),
@@ -275,6 +289,10 @@ pub async fn fetch_latency_breakdown(
             LatencyBreakdownSlice::IndexerDataServiceAllChunksSent,
             MetricSamples::new(indexer_data_service_all_chunks_sent_samples),
         );
+        samples.insert(
+            LatencyBreakdownSlice::IndexerProcessorLatency,
+            MetricSamples::new(indexer_processor_latency_samples),
+        );
     }
     Ok(LatencyBreakdown::new(samples))
 }
diff --git a/testsuite/forge/src/success_criteria.rs b/testsuite/forge/src/success_criteria.rs
@@ -79,7 +79,7 @@ impl MetricsThreshold {
         }
 
         if metrics.is_empty() {
-            bail!("Empty metrics provided");
+            bail!("Empty metrics provided for {}", metrics_name);
         }
         let breach_count = metrics
             .iter()