From 76d1704a1ecdb2d2ae4176f6e7c2dfa0c0f6069a Mon Sep 17 00:00:00 2001 From: Annie Liang <64233642+xinlian12@users.noreply.github.com> Date: Mon, 7 Feb 2022 16:37:02 -0800 Subject: [PATCH] AddSuportForLoggingEmptyPageDiagnostics- shortTermFix (#26869) * logEmptyPageDiagnostics Co-authored-by: annie-mac Co-authored-by: annie-mac --- .../azure/cosmos/implementation/Configs.java | 10 ++++ .../ImplementationBridgeHelpers.java | 2 + ...ParallelDocumentQueryExecutionContext.java | 46 ++++++++++++++++++- ...llelDocumentQueryExecutionContextBase.java | 4 -- .../models/CosmosQueryRequestOptions.java | 22 +++++++++ 5 files changed, 78 insertions(+), 6 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index ccf4e48ef5403..bf755bfcf754e 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -99,6 +99,10 @@ public class Configs { private static final String OPEN_CONNECTIONS_RETRIES_COUNT_NAME = "COSMOS.OPEN_CONNECTIONS_RETRIES_COUNT"; private static final int DEFAULT_OPEN_CONNECTIONS_RETRIES_COUNT = 1; + // whether to allow query empty page diagnostics logging + private static final String QUERY_EMPTY_PAGE_DIAGNOSTICS_ENABLED = "COSMOS.QUERY_EMPTY_PAGE_DIAGNOSTICS_ENABLED"; + private static final boolean DEFAULT_QUERY_EMPTY_PAGE_DIAGNOSTICS_ENABLED = false; + public Configs() { this.sslContext = sslContextInit(); } @@ -272,6 +276,12 @@ public static int getOpenConnectionsRetriesCount() { DEFAULT_OPEN_CONNECTIONS_RETRIES_COUNT); } + public static boolean isEmptyPageDiagnosticsEnabled() { + return getJVMConfigAsBoolean( + QUERY_EMPTY_PAGE_DIAGNOSTICS_ENABLED, + DEFAULT_QUERY_EMPTY_PAGE_DIAGNOSTICS_ENABLED); + } + private static int getJVMConfigAsInt(String propName, int defaultValue) { String propValue = System.getProperty(propName); return getIntValue(propValue, defaultValue); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java index 655a09b46ab85..e4fc9ef79120e 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java @@ -174,6 +174,8 @@ public interface CosmosQueryRequestOptionsAccessor { Map getHeader(CosmosQueryRequestOptions queryRequestOptions); boolean isQueryPlanRetrievalDisallowed(CosmosQueryRequestOptions queryRequestOptions); CosmosQueryRequestOptions disallowQueryPlanRetrieval(CosmosQueryRequestOptions queryRequestOptions); + boolean isEmptyPageDiagnosticsEnabled(CosmosQueryRequestOptions queryRequestOptions); + CosmosQueryRequestOptions setEmptyPageDiagnosticsEnabled(CosmosQueryRequestOptions queryRequestOptions, boolean emptyPageDiagnosticsEnabled); } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/query/ParallelDocumentQueryExecutionContext.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/query/ParallelDocumentQueryExecutionContext.java index fe6f55a3d4136..68a4e05de14d0 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/query/ParallelDocumentQueryExecutionContext.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/query/ParallelDocumentQueryExecutionContext.java @@ -5,10 +5,12 @@ import com.azure.cosmos.BridgeInternal; import com.azure.cosmos.CosmosDiagnostics; import com.azure.cosmos.CosmosException; +import com.azure.cosmos.implementation.ClientSideRequestStatistics; import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.DiagnosticsClientContext; import com.azure.cosmos.implementation.DocumentClientRetryPolicy; import com.azure.cosmos.implementation.HttpConstants; +import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.PartitionKeyRange; import com.azure.cosmos.implementation.QueryMetrics; import com.azure.cosmos.implementation.RequestChargeTracker; @@ -23,6 +25,9 @@ import com.azure.cosmos.models.FeedResponse; import com.azure.cosmos.models.ModelBridgeInternal; import com.azure.cosmos.models.SqlQuerySpec; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.util.concurrent.Queues; @@ -44,6 +49,8 @@ */ public class ParallelDocumentQueryExecutionContext extends ParallelDocumentQueryExecutionContextBase { + private static final Logger logger = LoggerFactory.getLogger(ParallelDocumentQueryExecutionContext.class); + private final CosmosQueryRequestOptions cosmosQueryRequestOptions; private final Map partitionKeyRangeToContinuationTokenMap; @@ -213,10 +220,11 @@ private static class EmptyPagesFilterTransformer private final RequestChargeTracker tracker; private DocumentProducer.DocumentProducerFeedResponse previousPage; private final CosmosQueryRequestOptions cosmosQueryRequestOptions; + private final UUID correlatedActivityId; private ConcurrentMap emptyPageQueryMetricsMap = new ConcurrentHashMap<>(); private CosmosDiagnostics cosmosDiagnostics; - public EmptyPagesFilterTransformer(RequestChargeTracker tracker, CosmosQueryRequestOptions options) { + public EmptyPagesFilterTransformer(RequestChargeTracker tracker, CosmosQueryRequestOptions options, UUID correlatedActivityId) { if (tracker == null) { throw new IllegalArgumentException("Request Charge Tracker must not be null."); @@ -225,6 +233,7 @@ public EmptyPagesFilterTransformer(RequestChargeTracker tracker, CosmosQueryRequ this.tracker = tracker; this.previousPage = null; this.cosmosQueryRequestOptions = options; + this.correlatedActivityId = correlatedActivityId; } private DocumentProducer.DocumentProducerFeedResponse plusCharge( @@ -286,6 +295,18 @@ public Flux> apply(Flux.DocumentProducerFeed BridgeInternal.queryMetricsFromFeedResponse(documentProducerFeedResponse.pageResult); QueryMetrics.mergeQueryMetricsMap(emptyPageQueryMetricsMap, currentQueryMetrics); cosmosDiagnostics = documentProducerFeedResponse.pageResult.getCosmosDiagnostics(); + + if (ImplementationBridgeHelpers + .CosmosQueryRequestOptionsHelper + .getCosmosQueryRequestOptionsAccessor() + .isEmptyPageDiagnosticsEnabled(cosmosQueryRequestOptions)) { + + logEmptyPageDiagnostics( + cosmosDiagnostics, + this.correlatedActivityId, + documentProducerFeedResponse.pageResult.getActivityId()); + } + return false; } return true; @@ -366,6 +387,27 @@ public Flux> apply(Flux.DocumentProducerFeed } } + static void logEmptyPageDiagnostics( + CosmosDiagnostics cosmosDiagnostics, + UUID correlatedActivityId, + String activityId) { + List requestStatistics = + BridgeInternal.getClientSideRequestStatisticsList(cosmosDiagnostics); + + try { + if (logger.isInfoEnabled()) { + logger.info( + "Empty page request diagnostics for correlatedActivityId [{}] - activityId [{}] - [{}]", + correlatedActivityId, + activityId, + Utils.getSimpleObjectMapper().writeValueAsString(requestStatistics)); + } + + } catch (JsonProcessingException e) { + logger.warn("Failed to log empty page diagnostics. ", e); + } + } + @Override public Flux> drainAsync( int maxPageSize) { @@ -386,7 +428,7 @@ public Flux> drainAsync( logger.debug("ParallelQuery: flux mergeSequential" + " concurrency {}, prefetch {}", fluxConcurrency, fluxPrefetch); return Flux.mergeSequential(obs, fluxConcurrency, fluxPrefetch) - .transformDeferred(new EmptyPagesFilterTransformer<>(new RequestChargeTracker(), this.cosmosQueryRequestOptions)); + .transformDeferred(new EmptyPagesFilterTransformer<>(new RequestChargeTracker(), this.cosmosQueryRequestOptions, correlatedActivityId)); } @Override diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/query/ParallelDocumentQueryExecutionContextBase.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/query/ParallelDocumentQueryExecutionContextBase.java index 5eb031a46c14b..7e70500d31767 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/query/ParallelDocumentQueryExecutionContextBase.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/query/ParallelDocumentQueryExecutionContextBase.java @@ -17,8 +17,6 @@ import com.azure.cosmos.models.FeedResponse; import com.azure.cosmos.models.PartitionKey; import com.azure.cosmos.models.SqlQuerySpec; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -37,7 +35,6 @@ public abstract class ParallelDocumentQueryExecutionContextBase extends DocumentQueryExecutionContextBase implements IDocumentQueryExecutionComponent { - protected final Logger logger; protected final List> documentProducers; protected final SqlQuerySpec querySpec; protected int pageSize; @@ -51,7 +48,6 @@ protected ParallelDocumentQueryExecutionContextBase(DiagnosticsClientContext dia super(diagnosticsClientContext, client, resourceTypeEnum, resourceType, query, cosmosQueryRequestOptions, resourceLink, getLazyFeedResponse, correlatedActivityId); - logger = LoggerFactory.getLogger(this.getClass()); documentProducers = new ArrayList<>(); if (!Strings.isNullOrEmpty(rewrittenQuery)) { this.querySpec = new SqlQuerySpec(rewrittenQuery, super.query.getParameters()); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosQueryRequestOptions.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosQueryRequestOptions.java index 7e24aed0ce7fc..295fe9ff4f59f 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosQueryRequestOptions.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosQueryRequestOptions.java @@ -4,6 +4,7 @@ package com.azure.cosmos.models; import com.azure.cosmos.ConsistencyLevel; +import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.spark.OperationContextAndListenerTuple; import com.azure.cosmos.util.Beta; @@ -39,12 +40,15 @@ public class CosmosQueryRequestOptions { private Map customOptions; private boolean indexMetricsEnabled; private boolean queryPlanRetrievalDisallowed; + private boolean emptyPageDiagnosticsEnabled; /** * Instantiates a new query request options. */ public CosmosQueryRequestOptions() { + this.queryMetricsEnabled = true; + this.emptyPageDiagnosticsEnabled = Configs.isEmptyPageDiagnosticsEnabled(); } /** @@ -72,6 +76,7 @@ public CosmosQueryRequestOptions() { this.customOptions = options.customOptions; this.indexMetricsEnabled = options.indexMetricsEnabled; this.queryPlanRetrievalDisallowed = options.queryPlanRetrievalDisallowed; + this.emptyPageDiagnosticsEnabled = options.emptyPageDiagnosticsEnabled; } void setOperationContextAndListenerTuple(OperationContextAndListenerTuple operationContextAndListenerTuple) { @@ -550,6 +555,13 @@ boolean isQueryPlanRetrievalDisallowed() { return this.queryPlanRetrievalDisallowed; } + boolean isEmptyPageDiagnosticsEnabled() { return this.emptyPageDiagnosticsEnabled; } + + CosmosQueryRequestOptions setEmptyPageDiagnosticsEnabled(boolean emptyPageDiagnosticsEnabled) { + this.emptyPageDiagnosticsEnabled = emptyPageDiagnosticsEnabled; + return this; + } + /////////////////////////////////////////////////////////////////////////////////////////// // the following helper/accessor only helps to access this class outside of this package.// /////////////////////////////////////////////////////////////////////////////////////////// @@ -591,6 +603,16 @@ public CosmosQueryRequestOptions disallowQueryPlanRetrieval( public boolean isQueryPlanRetrievalDisallowed(CosmosQueryRequestOptions queryRequestOptions) { return queryRequestOptions.isQueryPlanRetrievalDisallowed(); } + + @Override + public boolean isEmptyPageDiagnosticsEnabled(CosmosQueryRequestOptions queryRequestOptions) { + return queryRequestOptions.isEmptyPageDiagnosticsEnabled(); + } + + @Override + public CosmosQueryRequestOptions setEmptyPageDiagnosticsEnabled(CosmosQueryRequestOptions queryRequestOptions, boolean emptyPageDiagnosticsEnabled) { + return queryRequestOptions.setEmptyPageDiagnosticsEnabled(emptyPageDiagnosticsEnabled); + } }); } }