From f36292435ee9f1f399f3313e6bb86a854e43e0b3 Mon Sep 17 00:00:00 2001 From: Arpita Date: Fri, 25 Sep 2020 10:14:06 +0530 Subject: [PATCH 1/7] Publish shard state metrics --- .../PerformanceAnalyzerPlugin.java | 2 + .../collectors/ShardStateCollector.java | 132 ++++++++++++++++++ .../performanceanalyzer/util/Utils.java | 3 +- .../collectors/ShardStateCollectorTest.java | 36 +++++ 4 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java create mode 100644 src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java index ea218014..291b58c4 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java @@ -15,6 +15,7 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ShardStateCollector; import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper; import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.handler.ConfigOverridesClusterSettingHandler; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.CacheConfigMetricsCollector; @@ -195,6 +196,7 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa NodeStatsFixedShardsMetricsCollector(performanceAnalyzerController)); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MasterServiceMetrics()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MasterServiceEventMetrics()); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new ShardStateCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new DisksCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new NetworkInterfaceCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(StatsCollector.instance()); diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java new file mode 100644 index 00000000..babf7206 --- /dev/null +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java @@ -0,0 +1,132 @@ +package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.jooq.tools.StringUtils; + +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics.SHARD_PRIMARY; +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics.SHARD_REPLICA; + +public class ShardStateCollector extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor { + public static final int SAMPLING_TIME_INTERVAL = MetricsConfiguration.CONFIG_MAP.get(ShardStateCollector.class).samplingInterval; + private static final Logger LOG = LogManager.getLogger(ShardStateCollector.class); + private static final int KEYS_PATH_LENGTH = 0; + private StringBuilder value; + + public ShardStateCollector() { + super(SAMPLING_TIME_INTERVAL, "ShardsStateCollector"); + value = new StringBuilder(); + } + + @Override + void collectMetrics( long startTime) { + if (ESResources.INSTANCE.getClusterService() == null) { + return; + } + ClusterState clusterState = ESResources.INSTANCE.getClusterService().state(); + + try { + value.setLength(0); + value.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds()) + .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); + for(ShardRouting shard : clusterState.routingTable().allShards()) { + String nodeName = StringUtils.EMPTY; + if (shard.assignedToNode()) { + nodeName = clusterState.nodes().get(shard.currentNodeId()).getName(); + } + value + .append(new ShardStateMetrics( + shard.getIndexName(), + shard.getId(), + shard.primary()?SHARD_PRIMARY:SHARD_REPLICA, + nodeName, + shard.state() == ShardRoutingState.STARTED?1:0, + shard.state() == ShardRoutingState.INITIALIZING?1:0, + shard.state() == ShardRoutingState.UNASSIGNED?1:0) + .serialize()) + .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); + } + saveMetricValues(value.toString(), startTime); + + } catch (Exception ex) { + LOG.debug("Exception in Collecting Shard Metrics: {} for startTime {}", () -> ex.toString(), + () -> startTime); + } + } + + @Override + public String getMetricsPath(long startTime, String... keysPath) { + if (keysPath.length != KEYS_PATH_LENGTH) { + throw new RuntimeException("keys length should be " + KEYS_PATH_LENGTH); + } + + return PerformanceAnalyzerMetrics.generatePath(startTime, PerformanceAnalyzerMetrics.sShardStatePath); + } + + public static class ShardStateMetrics extends MetricStatus { + + private final String indexName; + private final int shardId; + private final String shardType; + private final String nodeName; + private final int activeShardState; + private final int initializingShardState; + private final int unassignedShardState; + + public ShardStateMetrics(String indexName, int shardId, String shardType, String nodeName, int activeShardState, + int initializingShardState, int unassignedShardState) { + this.indexName = indexName; + this.shardId = shardId; + this.shardType = shardType; + this.nodeName = nodeName; + this.activeShardState = activeShardState; + this.initializingShardState = initializingShardState; + this.unassignedShardState = unassignedShardState; + } + + @JsonProperty(AllMetrics.CommonDimension.Constants.INDEX_NAME_VALUE) + public String getIndexName() { + return indexName; + } + + @JsonProperty(AllMetrics.CommonDimension.Constants.SHARDID_VALUE) + public int getShardId() { + return shardId; + } + + @JsonProperty(AllMetrics.ShardStateDimension.Constants.SHARD_TYPE) + public String getShardType() { + return shardType; + } + + @JsonProperty(AllMetrics.ShardStateDimension.Constants.NODE_NAME) + public String getNodeName() { + return nodeName; + } + + @JsonProperty(AllMetrics.ShardStateValue.Constants.SHARD_STATE_ACTIVE) + public int getActiveShardState() { + return activeShardState; + } + + @JsonProperty(AllMetrics.ShardStateValue.Constants.SHARD_STATE_INITIALIZING) + public int getInitializingShardState() { + return initializingShardState; + } + + @JsonProperty(AllMetrics.ShardStateValue.Constants.SHARD_STATE_UNASSIGNED) + public int getUnassignedShardState() { + return unassignedShardState; + } + } + +} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/Utils.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/Utils.java index 4c5b09e3..f59edbc7 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/Utils.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/Utils.java @@ -17,6 +17,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.CacheConfigMetricsCollector; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ShardStateCollector; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.CircuitBreakerCollector; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceEventMetrics; @@ -51,7 +52,7 @@ public static void configureMetrics() { MetricsConfiguration.CONFIG_MAP.put(NodeStatsFixedShardsMetricsCollector.class, cdefault); MetricsConfiguration.CONFIG_MAP.put(MasterServiceEventMetrics.class, new MetricsConfiguration.MetricConfig(1000, 0, 0)); MetricsConfiguration.CONFIG_MAP.put(MasterServiceMetrics.class, cdefault); - + MetricsConfiguration.CONFIG_MAP.put(ShardStateCollector.class, cdefault); } // These methods are utility functions for the Node Stat Metrics Collectors. These methods are used by both the all diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java new file mode 100644 index 00000000..ae9a2951 --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java @@ -0,0 +1,36 @@ +package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@Ignore +public class ShardStateCollectorTest extends CustomMetricsLocationTestBase { + + @Test + public void testShardsStateMetrics() { + MetricsConfiguration.CONFIG_MAP.put(ShardStateCollector.class, MetricsConfiguration.cdefault); + System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); + long startTimeInMills = 1153721339; + ShardStateCollector shardsStateCollector = new ShardStateCollector(); + shardsStateCollector.saveMetricValues("shard_state_metrics", startTimeInMills); + String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PluginSettings.instance().getMetricsLocation() + + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/shard_state_metrics/"); + PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation() + + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)); + assertEquals("shard_state_metrics", fetchedValue); + + try { + shardsStateCollector.saveMetricValues("shard_state_metrics", startTimeInMills, "123"); + assertTrue("Negative scenario test: Should have been a RuntimeException", true); + } catch (RuntimeException ex) { + //- expecting exception...1 values passed; 0 expected + } + } +} \ No newline at end of file From a7b677b7a055878806225086761a6d2a283d776f Mon Sep 17 00:00:00 2001 From: Arpita Date: Tue, 13 Oct 2020 13:47:29 +0530 Subject: [PATCH 2/7] Publish Shard State Metrics2 --- .../PerformanceAnalyzerPlugin.java | 2 +- .../collectors/ShardStateCollector.java | 91 +++++++++---------- .../collectors/ShardStateCollectorTest.java | 5 +- 3 files changed, 48 insertions(+), 50 deletions(-) diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java index 291b58c4..d1eaeb8a 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java @@ -196,10 +196,10 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa NodeStatsFixedShardsMetricsCollector(performanceAnalyzerController)); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MasterServiceMetrics()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MasterServiceEventMetrics()); - scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new ShardStateCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new DisksCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new NetworkInterfaceCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(StatsCollector.instance()); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new ShardStateCollector()); scheduledMetricCollectorsExecutor.start(); EventLog eventLog = new EventLog(); diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java index babf7206..9f75390e 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java @@ -9,9 +9,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.jooq.tools.StringUtils; +import org.jooq.tools.json.JSONObject; + +import java.util.List; import static com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics.SHARD_PRIMARY; import static com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics.SHARD_REPLICA; @@ -33,29 +37,39 @@ void collectMetrics( long startTime) { return; } ClusterState clusterState = ESResources.INSTANCE.getClusterService().state(); - + boolean inActiveShard = false; try { value.setLength(0); value.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds()) - .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); - for(ShardRouting shard : clusterState.routingTable().allShards()) { - String nodeName = StringUtils.EMPTY; - if (shard.assignedToNode()) { - nodeName = clusterState.nodes().get(shard.currentNodeId()).getName(); + .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); + RoutingTable routingTable = clusterState.routingTable(); + String[] indices = routingTable.indicesRouting().keys().toArray(String.class); + for (String index : indices) { + List allShardsIndex = routingTable.allShards(index); + value.append(createJsonObject(AllMetrics.ShardStateDimension.INDEX_NAME.toString(), index)); + for (ShardRouting shard :allShardsIndex) { + String nodeName = StringUtils.EMPTY; + if (shard.assignedToNode()) { + nodeName = clusterState.nodes().get(shard.currentNodeId()).getName(); + } + if (shard.state() != ShardRoutingState.STARTED) { + inActiveShard = true; + value + .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) + .append(new ShardStateMetrics( + shard.getId(), + shard.primary() ? SHARD_PRIMARY : SHARD_REPLICA, + nodeName, + shard.state().name()) + .serialize()); + + } } - value - .append(new ShardStateMetrics( - shard.getIndexName(), - shard.getId(), - shard.primary()?SHARD_PRIMARY:SHARD_REPLICA, - nodeName, - shard.state() == ShardRoutingState.STARTED?1:0, - shard.state() == ShardRoutingState.INITIALIZING?1:0, - shard.state() == ShardRoutingState.UNASSIGNED?1:0) - .serialize()) - .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); } - saveMetricValues(value.toString(), startTime); + value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); + if(inActiveShard) { + saveMetricValues(value.toString(), startTime); + } } catch (Exception ex) { LOG.debug("Exception in Collecting Shard Metrics: {} for startTime {}", () -> ex.toString(), @@ -63,6 +77,13 @@ void collectMetrics( long startTime) { } } + @SuppressWarnings("unchecked") + private String createJsonObject(String key, String value) { + JSONObject json = new JSONObject(); + json.put(key,value); + return json.toString(); + } + @Override public String getMetricsPath(long startTime, String... keysPath) { if (keysPath.length != KEYS_PATH_LENGTH) { @@ -74,28 +95,16 @@ public String getMetricsPath(long startTime, String... keysPath) { public static class ShardStateMetrics extends MetricStatus { - private final String indexName; private final int shardId; private final String shardType; private final String nodeName; - private final int activeShardState; - private final int initializingShardState; - private final int unassignedShardState; + private final String shardState; - public ShardStateMetrics(String indexName, int shardId, String shardType, String nodeName, int activeShardState, - int initializingShardState, int unassignedShardState) { - this.indexName = indexName; + public ShardStateMetrics(int shardId, String shardType, String nodeName, String shardState) { this.shardId = shardId; this.shardType = shardType; this.nodeName = nodeName; - this.activeShardState = activeShardState; - this.initializingShardState = initializingShardState; - this.unassignedShardState = unassignedShardState; - } - - @JsonProperty(AllMetrics.CommonDimension.Constants.INDEX_NAME_VALUE) - public String getIndexName() { - return indexName; + this.shardState = shardState; } @JsonProperty(AllMetrics.CommonDimension.Constants.SHARDID_VALUE) @@ -113,19 +122,9 @@ public String getNodeName() { return nodeName; } - @JsonProperty(AllMetrics.ShardStateValue.Constants.SHARD_STATE_ACTIVE) - public int getActiveShardState() { - return activeShardState; - } - - @JsonProperty(AllMetrics.ShardStateValue.Constants.SHARD_STATE_INITIALIZING) - public int getInitializingShardState() { - return initializingShardState; - } - - @JsonProperty(AllMetrics.ShardStateValue.Constants.SHARD_STATE_UNASSIGNED) - public int getUnassignedShardState() { - return unassignedShardState; + @JsonProperty(AllMetrics.ShardStateValue.Constants.SHARD_STATE) + public String getShardState() { + return shardState; } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java index ae9a2951..8da8fb92 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java @@ -4,13 +4,11 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -@Ignore public class ShardStateCollectorTest extends CustomMetricsLocationTestBase { @Test @@ -33,4 +31,5 @@ public void testShardsStateMetrics() { //- expecting exception...1 values passed; 0 expected } } -} \ No newline at end of file +} + From 4f460e172227a8de218f687446780f7794b523df Mon Sep 17 00:00:00 2001 From: Arpita Date: Tue, 20 Oct 2020 00:13:42 +0530 Subject: [PATCH 3/7] Publish Shard State Metrics --- .../collectors/ShardStateCollector.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java index 9f75390e..7bfbb9ea 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java @@ -1,10 +1,13 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.metrics.ExceptionsAndErrors; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.metrics.WriterMetrics; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -33,6 +36,7 @@ public ShardStateCollector() { @Override void collectMetrics( long startTime) { + long mCurrT = System.currentTimeMillis(); if (ESResources.INSTANCE.getClusterService() == null) { return; } @@ -70,8 +74,12 @@ void collectMetrics( long startTime) { if(inActiveShard) { saveMetricValues(value.toString(), startTime); } - + PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat( + WriterMetrics.SHARD_STATE_COLLECTOR_EXECUTION_TIME, "", + System.currentTimeMillis() - mCurrT); } catch (Exception ex) { + PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat( + ExceptionsAndErrors.SHARD_STATE_COLLECTOR_ERROR, "", 1); LOG.debug("Exception in Collecting Shard Metrics: {} for startTime {}", () -> ex.toString(), () -> startTime); } From 34e68411918547f44f7727ba58247d47389570b7 Mon Sep 17 00:00:00 2001 From: Arpita Date: Tue, 20 Oct 2020 01:12:59 +0530 Subject: [PATCH 4/7] Publish Shard State Metrics --- .../performanceanalyzer/collectors/ShardStateCollector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java index 7bfbb9ea..2c06a714 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java @@ -74,7 +74,7 @@ void collectMetrics( long startTime) { if(inActiveShard) { saveMetricValues(value.toString(), startTime); } - PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat( + PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat( WriterMetrics.SHARD_STATE_COLLECTOR_EXECUTION_TIME, "", System.currentTimeMillis() - mCurrT); } catch (Exception ex) { From b01a6925690d3e1ef9213c73e30552ede780142f Mon Sep 17 00:00:00 2001 From: Arpita Date: Tue, 20 Oct 2020 14:39:45 +0530 Subject: [PATCH 5/7] Publish Shard State Metrics --- .../collectors/ShardStateCollector.java | 21 ++++++++++++++++--- .../collectors/ShardStateCollectorTest.java | 15 +++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java index 2c06a714..c2c04a8e 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java @@ -1,3 +1,18 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; @@ -20,8 +35,8 @@ import java.util.List; -import static com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics.SHARD_PRIMARY; -import static com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics.SHARD_REPLICA; +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardType.SHARD_PRIMARY; +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardType.SHARD_REPLICA; public class ShardStateCollector extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor { public static final int SAMPLING_TIME_INTERVAL = MetricsConfiguration.CONFIG_MAP.get(ShardStateCollector.class).samplingInterval; @@ -62,7 +77,7 @@ void collectMetrics( long startTime) { .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) .append(new ShardStateMetrics( shard.getId(), - shard.primary() ? SHARD_PRIMARY : SHARD_REPLICA, + shard.primary() ? SHARD_PRIMARY.toString() : SHARD_REPLICA.toString(), nodeName, shard.state().name()) .serialize()); diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java index 8da8fb92..ce522f7a 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java @@ -1,3 +1,18 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; From e595828701dee938fd6065f223c34e9fba5e3609 Mon Sep 17 00:00:00 2001 From: Arpita Date: Fri, 23 Oct 2020 03:07:41 +0530 Subject: [PATCH 6/7] Publish Shard State Metrics --- .../PerformanceAnalyzerPlugin.java | 3 ++- .../collectors/ShardStateCollector.java | 12 +++++++++++- .../config/PerformanceAnalyzerController.java | 11 +++++++++++ .../collectors/ShardStateCollectorTest.java | 9 ++++++++- 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java index d1eaeb8a..49b16a87 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java @@ -199,7 +199,8 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new DisksCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new NetworkInterfaceCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(StatsCollector.instance()); - scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new ShardStateCollector()); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new ShardStateCollector( + performanceAnalyzerController,configOverridesWrapper)); scheduledMetricCollectorsExecutor.start(); EventLog eventLog = new EventLog(); diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java index c2c04a8e..1d35cae1 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollector.java @@ -17,6 +17,8 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PerformanceAnalyzerController; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor; @@ -42,15 +44,23 @@ public class ShardStateCollector extends PerformanceAnalyzerMetricsCollector imp public static final int SAMPLING_TIME_INTERVAL = MetricsConfiguration.CONFIG_MAP.get(ShardStateCollector.class).samplingInterval; private static final Logger LOG = LogManager.getLogger(ShardStateCollector.class); private static final int KEYS_PATH_LENGTH = 0; + private final ConfigOverridesWrapper configOverridesWrapper; + private final PerformanceAnalyzerController controller; private StringBuilder value; - public ShardStateCollector() { + public ShardStateCollector(PerformanceAnalyzerController controller, + ConfigOverridesWrapper configOverridesWrapper) { super(SAMPLING_TIME_INTERVAL, "ShardsStateCollector"); value = new StringBuilder(); + this.controller = controller; + this.configOverridesWrapper = configOverridesWrapper; } @Override void collectMetrics( long startTime) { + if(!controller.isCollectorEnabled(configOverridesWrapper, getCollectorName())) { + return; + } long mCurrT = System.currentTimeMillis(); if (ESResources.INSTANCE.getClusterService() == null) { return; diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerController.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerController.java index b752a96d..128a4ba1 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerController.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerController.java @@ -5,8 +5,10 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import java.util.Scanner; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -257,4 +259,13 @@ private void saveStateToConf(boolean featureEnabled, String fileName) { } }); } + + public boolean isCollectorEnabled(ConfigOverridesWrapper configOverridesWrapper, String collectorName) { + List enabledCollectorsList = configOverridesWrapper.getCurrentClusterConfigOverrides().getEnable() + .getCollectors(); + if(enabledCollectorsList.contains(collectorName)) { + return true; + } + return false; + } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java index ce522f7a..9239d3d7 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTest.java @@ -16,10 +16,13 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PerformanceAnalyzerController; import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import org.junit.Test; +import org.mockito.Mockito; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -31,7 +34,11 @@ public void testShardsStateMetrics() { MetricsConfiguration.CONFIG_MAP.put(ShardStateCollector.class, MetricsConfiguration.cdefault); System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); long startTimeInMills = 1153721339; - ShardStateCollector shardsStateCollector = new ShardStateCollector(); + PerformanceAnalyzerController controller = Mockito.mock(PerformanceAnalyzerController.class); + ConfigOverridesWrapper configOverrides = Mockito.mock(ConfigOverridesWrapper.class); + Mockito.when(controller.isCollectorEnabled(configOverrides, "ShardStateCollector")) + .thenReturn(true); + ShardStateCollector shardsStateCollector = new ShardStateCollector(controller, configOverrides); shardsStateCollector.saveMetricValues("shard_state_metrics", startTimeInMills); String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/shard_state_metrics/"); From c6d5e7c5db527253414363aeda734b36d3f3b0b8 Mon Sep 17 00:00:00 2001 From: Arpita Date: Fri, 23 Oct 2020 03:10:56 +0530 Subject: [PATCH 7/7] Publish Shard State Metrics --- .../config/PerformanceAnalyzerController.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerController.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerController.java index 128a4ba1..4478d439 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerController.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerController.java @@ -261,11 +261,11 @@ private void saveStateToConf(boolean featureEnabled, String fileName) { } public boolean isCollectorEnabled(ConfigOverridesWrapper configOverridesWrapper, String collectorName) { + if(configOverridesWrapper == null) { + return false; + } List enabledCollectorsList = configOverridesWrapper.getCurrentClusterConfigOverrides().getEnable() .getCollectors(); - if(enabledCollectorsList.contains(collectorName)) { - return true; - } - return false; + return enabledCollectorsList.contains(collectorName) ? true: false; } }