From 85be081ecfa961337602b00594200050ac1f7c5e Mon Sep 17 00:00:00 2001 From: larry-aptos <112209412+larry-aptos@users.noreply.github.com> Date: Mon, 2 Oct 2023 12:39:31 -0700 Subject: [PATCH] [indexer][api] update the metrcis for api gateway consumption. (#10322) --- .../indexer-grpc-data-service/src/metrics.rs | 27 +++++++++----- .../indexer-grpc-data-service/src/service.rs | 35 +++++++++++++------ 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/ecosystem/indexer-grpc/indexer-grpc-data-service/src/metrics.rs b/ecosystem/indexer-grpc/indexer-grpc-data-service/src/metrics.rs index c3c26a57a8cb7..e5f1655179877 100644 --- a/ecosystem/indexer-grpc/indexer-grpc-data-service/src/metrics.rs +++ b/ecosystem/indexer-grpc/indexer-grpc-data-service/src/metrics.rs @@ -10,9 +10,9 @@ use once_cell::sync::Lazy; /// Latest processed transaction version. pub static LATEST_PROCESSED_VERSION: Lazy = Lazy::new(|| { register_int_gauge_vec!( - "indexer_grpc_data_service_latest_processed_version", + "indexer_grpc_data_service_with_user_latest_processed_version", "Latest processed transaction version", - &["request_token", "processor_name"], + &["request_token", "email"], ) .unwrap() }); @@ -20,9 +20,9 @@ pub static LATEST_PROCESSED_VERSION: Lazy = Lazy::new(|| { /// Number of transactions that served by data service. pub static PROCESSED_VERSIONS_COUNT: Lazy = Lazy::new(|| { register_int_counter_vec!( - "indexer_grpc_data_service_processed_versions", + "indexer_grpc_data_service_with_user_processed_versions", "Number of transactions that have been processed by data service", - &["request_token", "processor_name"], + &["request_token", "email"], ) .unwrap() }); @@ -40,9 +40,9 @@ pub static ERROR_COUNT: Lazy = Lazy::new(|| { /// Data latency for data service based on latest processed transaction based on selected processor. pub static PROCESSED_LATENCY_IN_SECS: Lazy = Lazy::new(|| { register_gauge_vec!( - "indexer_grpc_data_service_latest_data_latency_in_secs", + "indexer_grpc_data_service_with_user_latest_data_latency_in_secs", "Latency of data service based on latest processed transaction", - &["request_token", "processor_name"], + &["request_token", "email"], ) .unwrap() }); @@ -60,9 +60,9 @@ pub static PROCESSED_LATENCY_IN_SECS_ALL: Lazy = Lazy::new(|| { /// Number of transactions in each batch that data service has processed. pub static PROCESSED_BATCH_SIZE: Lazy = Lazy::new(|| { register_int_gauge_vec!( - "indexer_grpc_data_service_processed_batch_size", + "indexer_grpc_data_service_with_user_processed_batch_size", "Size of latest processed batch by data service", - &["request_token", "processor_name"], + &["request_token", "email"], ) .unwrap() }); @@ -84,3 +84,14 @@ pub static SHORT_CONNECTION_COUNT: Lazy = Lazy::new(|| { ) .unwrap() }); + +/// Count of bytes transfered to the client. This only represents the bytes prepared and ready +/// to send to the client. It does not represent the bytes actually sent to the client. +pub static BYTES_READY_TO_TRANSFER_FROM_SERVER: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "indexer_grpc_data_service_bytes_ready_to_transfer_from_server", + "Count of bytes ready to transfer to the client", + &["request_token", "email"], + ) + .unwrap() +}); diff --git a/ecosystem/indexer-grpc/indexer-grpc-data-service/src/service.rs b/ecosystem/indexer-grpc/indexer-grpc-data-service/src/service.rs index cbc7300755993..31dfbe6bb89f8 100644 --- a/ecosystem/indexer-grpc/indexer-grpc-data-service/src/service.rs +++ b/ecosystem/indexer-grpc/indexer-grpc-data-service/src/service.rs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use crate::metrics::{ - CONNECTION_COUNT, ERROR_COUNT, LATEST_PROCESSED_VERSION, PROCESSED_BATCH_SIZE, - PROCESSED_LATENCY_IN_SECS, PROCESSED_LATENCY_IN_SECS_ALL, PROCESSED_VERSIONS_COUNT, - SHORT_CONNECTION_COUNT, + BYTES_READY_TO_TRANSFER_FROM_SERVER, CONNECTION_COUNT, ERROR_COUNT, LATEST_PROCESSED_VERSION, + PROCESSED_BATCH_SIZE, PROCESSED_LATENCY_IN_SECS, PROCESSED_LATENCY_IN_SECS_ALL, + PROCESSED_VERSIONS_COUNT, SHORT_CONNECTION_COUNT, }; use anyhow::Context; use aptos_indexer_grpc_utils::{ @@ -261,6 +261,19 @@ impl RawData for RawDataServerWrapper { transactions_count = Some(count - transaction_data.len() as u64); } }; + // Note: this is not the actual bytes transferred to the client. + // This is the bytes consumed internally by the server + // and ready to be transferred to the client. + let bytes_ready_to_transfer = transaction_data + .iter() + .map(|(encoded, _)| encoded.len()) + .sum::(); + BYTES_READY_TO_TRANSFER_FROM_SERVER + .with_label_values(&[ + request_metadata.request_token.as_str(), + request_metadata.request_email.as_str(), + ]) + .inc_by(bytes_ready_to_transfer as u64); // 2. Push the data to the response channel, i.e. stream the data to the client. let current_batch_size = transaction_data.as_slice().len(); let end_of_batch_version = transaction_data.as_slice().last().unwrap().1; @@ -280,20 +293,20 @@ impl RawData for RawDataServerWrapper { Ok(_) => { PROCESSED_BATCH_SIZE .with_label_values(&[ - request_metadata.request_user_classification.as_str(), - request_metadata.request_name.as_str(), + request_metadata.request_token.as_str(), + request_metadata.request_email.as_str(), ]) .set(current_batch_size as i64); LATEST_PROCESSED_VERSION .with_label_values(&[ - request_metadata.request_user_classification.as_str(), - request_metadata.request_name.as_str(), + request_metadata.request_token.as_str(), + request_metadata.request_email.as_str(), ]) .set(end_of_batch_version as i64); PROCESSED_VERSIONS_COUNT .with_label_values(&[ - request_metadata.request_user_classification.as_str(), - request_metadata.request_name.as_str(), + request_metadata.request_token.as_str(), + request_metadata.request_email.as_str(), ]) .inc_by(current_batch_size as u64); if let Some(data_latency_in_secs) = data_latency_in_secs { @@ -302,8 +315,8 @@ impl RawData for RawDataServerWrapper { if current_batch_size % BLOB_STORAGE_SIZE != 0 { PROCESSED_LATENCY_IN_SECS .with_label_values(&[ - request_metadata.request_user_classification.as_str(), - request_metadata.request_name.as_str(), + request_metadata.request_token.as_str(), + request_metadata.request_email.as_str(), ]) .set(data_latency_in_secs); PROCESSED_LATENCY_IN_SECS_ALL