diff --git a/build.gradle b/build.gradle index 15d4624d..73b048e5 100644 --- a/build.gradle +++ b/build.gradle @@ -147,6 +147,7 @@ jacocoTestReport { exclude: [ '**/FaultDetectionMetricsCollector.class', '**/MasterThrottlingMetricsCollector.class', + '**/ClusterSettingsManager.class', ]) }) } @@ -167,7 +168,7 @@ jacocoTestCoverageVerification { violationRules { rule { limit { - minimum = 0.6212 + minimum = 0.8 } } } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceEventMetrics.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceEventMetrics.java index 6bf68c75..5973bdfd 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceEventMetrics.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceEventMetrics.java @@ -23,6 +23,7 @@ import java.util.Queue; import java.util.concurrent.ThreadPoolExecutor; +import com.google.common.annotations.VisibleForTesting; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.cluster.service.MasterService; @@ -43,7 +44,6 @@ public class MasterServiceEventMetrics extends PerformanceAnalyzerMetricsCollect MasterServiceEventMetrics.class).samplingInterval; private static final Logger LOG = LogManager.getLogger(MasterServiceEventMetrics.class); private static final String MASTER_NODE_NOT_UP_METRIC = "MasterNodeNotUp"; - private long lastTaskInsertionOrder; private static final int KEYS_PATH_LENGTH = 3; private StringBuilder value; private static final int TPEXECUTOR_ADD_PENDING_PARAM_COUNT = 3; @@ -53,6 +53,9 @@ public class MasterServiceEventMetrics extends PerformanceAnalyzerMetricsCollect private long currentThreadId; private Object currentWorker; + @VisibleForTesting + long lastTaskInsertionOrder; + public MasterServiceEventMetrics() { super(SAMPLING_TIME_INTERVAL, "MasterServiceEventMetrics"); masterServiceCurrentQueue = null; @@ -137,7 +140,8 @@ public void collectMetrics(long startTime) { } } - private void generateFinishMetrics(long startTime) { + @VisibleForTesting + void generateFinishMetrics(long startTime) { if (lastTaskInsertionOrder != -1) { value.append(PerformanceAnalyzerMetrics.getCurrentTimeMetric()); PerformanceAnalyzerMetrics.addMetricEntry(value, MasterMetricValues.FINISH_TIME.toString(), diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollector.java index bdebdfc4..86e9f031 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollector.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollector.java @@ -237,42 +237,6 @@ public NodeStatsMetricsFixedShardsPerCollectionStatus(ShardStats shardStats) { this.shardSizeInBytes = calculate(ShardStatsValue.SHARD_SIZE_IN_BYTES); } - @SuppressWarnings("checkstyle:parameternumber") - public NodeStatsMetricsFixedShardsPerCollectionStatus(long indexingThrottleTime, long refreshCount, long refreshTime, - long flushCount, long flushTime, long mergeCount, - long mergeTime, long mergeCurrent, long indexBufferBytes, - long segmentCount, long segmentsMemory, long termsMemory, - long storedFieldsMemory, long termVectorsMemory, - long normsMemory, long pointsMemory, long docValuesMemory, - long indexWriterMemory, long versionMapMemory, - long bitsetMemory, long shardSizeInBytes) { - super(); - this.shardStats = null; - - this.indexingThrottleTime = indexingThrottleTime; - this.refreshCount = refreshCount; - this.refreshTime = refreshTime; - this.flushCount = flushCount; - this.flushTime = flushTime; - this.mergeCount = mergeCount; - this.mergeTime = mergeTime; - this.mergeCurrent = mergeCurrent; - this.indexBufferBytes = indexBufferBytes; - this.segmentCount = segmentCount; - this.segmentsMemory = segmentsMemory; - this.termsMemory = termsMemory; - this.storedFieldsMemory = storedFieldsMemory; - this.termVectorsMemory = termVectorsMemory; - this.normsMemory = normsMemory; - this.pointsMemory = pointsMemory; - this.docValuesMemory = docValuesMemory; - this.indexWriterMemory = indexWriterMemory; - this.versionMapMemory = versionMapMemory; - this.bitsetMemory = bitsetMemory; - this.shardSizeInBytes = shardSizeInBytes; - } - - private long calculate(ShardStatsValue nodeMetric) { return valueCalculators.get(nodeMetric.toString()).calculateValue(shardStats); } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerController.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerController.java index 4478d439..9792435a 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerController.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerController.java @@ -1,5 +1,10 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.config; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerPlugin; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ScheduledMetricCollectorsExecutor; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import java.io.File; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -7,16 +12,9 @@ import java.nio.file.Paths; import java.util.List; import java.util.Scanner; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides.ConfigOverridesWrapper; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerPlugin; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ScheduledMetricCollectorsExecutor; - public class PerformanceAnalyzerController { private static final String PERFORMANCE_ANALYZER_ENABLED_CONF = "performance_analyzer_enabled.conf"; private static final String RCA_ENABLED_CONF = "rca_enabled.conf"; diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/whoami/WhoAmIResponse.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/whoami/WhoAmIResponse.java index 4b8f8886..2ea421ca 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/whoami/WhoAmIResponse.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/whoami/WhoAmIResponse.java @@ -25,8 +25,10 @@ public class WhoAmIResponse extends ActionResponse implements ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("whoami"); - builder.endObject(); + builder + .startObject() + .field("whoami", "whoami") + .endObject(); return builder; } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListener.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListener.java index 75f8df3b..184bf940 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListener.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListener.java @@ -43,7 +43,7 @@ public PerformanceAnalyzerSearchListener(final PerformanceAnalyzerController con @Override public String toString() { - return "PerformanceAnalyzerSearchListener"; + return PerformanceAnalyzerSearchListener.class.getSimpleName(); } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionFilterTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionFilterTests.java index 091b96a3..dc737211 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionFilterTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionFilterTests.java @@ -15,6 +15,7 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.action; +import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; @@ -40,6 +41,7 @@ public class PerformanceAnalyzerActionFilterTests { @Mock private PerformanceAnalyzerController controller; @Mock private SearchRequest searchRequest; @Mock private BulkRequest bulkRequest; + @Mock private ActionRequest request; @Mock private ActionListener listener; @Mock private ActionFilterChain chain; @Mock private Task task; @@ -63,9 +65,19 @@ public void testApplyWithBulkRequest() { testApply(bulkRequest); } + @Test + public void testApplyWithOtherRequest() { + testApply(request); + } + private void testApply(ActionRequest request) { filter.apply(task, "_action", request, listener, chain); verify(chain).proceed(eq(task), eq("_action"), eq(request), any()); } + + @Test + public void testOrder() { + assertEquals(Integer.MIN_VALUE, filter.order()); + } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionListenerTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionListenerTests.java index c3cdb071..d5c9ae57 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionListenerTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionListenerTests.java @@ -1,5 +1,5 @@ /* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright <2020> Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -15,49 +15,106 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.action; -import org.junit.Ignore; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import static org.junit.Assert.assertEquals; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.Event; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.TestUtil; +import java.util.List; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.rest.RestStatus; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +@SuppressWarnings("unchecked") +public class PerformanceAnalyzerActionListenerTests { + private static final String listenerId= "12345"; + private long startTimeInMills = 1153721339; + private PerformanceAnalyzerActionListener actionListener; + private ActionListener originalActionListener; + + @Before + public void init() { + originalActionListener = Mockito.mock(ActionListener.class); + actionListener = new PerformanceAnalyzerActionListener(); + } + + @Test + public void testGetMetricsPath() { + String expectedPath = PluginSettings.instance().getMetricsLocation() + + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills) + "/" + + PerformanceAnalyzerMetrics.sThreadsPath + "/" + + PerformanceAnalyzerMetrics.sHttpPath + "/" + + "bulk/bulkId/start"; + String actualPath = actionListener.getMetricsPath(startTimeInMills, "bulk", "bulkId", "start"); + assertEquals(expectedPath, actualPath); + + try { + actionListener.getMetricsPath(startTimeInMills, "bulk"); + fail("Negative scenario test: Should have been a RuntimeException"); + } catch (RuntimeException ex) { + //- expecting exception...1 values passed; 3 expected + } + } + + @Test + public void testOnResponseWithBulkResponse() { + BulkResponse bulkResponse = Mockito.mock(BulkResponse.class); + Mockito.when(bulkResponse.status()).thenReturn(RestStatus.OK); + actionListener.set(RequestType.bulk, listenerId , originalActionListener); + testOnResponse(bulkResponse); + } + + @Test + public void testOnResponseWithSearchResponse() { + SearchResponse searchResponse = Mockito.mock(SearchResponse.class); + Mockito.when(searchResponse.status()).thenReturn(RestStatus.OK); + actionListener.set(RequestType.search, listenerId , originalActionListener); + testOnResponse(searchResponse); + } -@Ignore -public class PerformanceAnalyzerActionListenerTests extends CustomMetricsLocationTestBase { + @Test + public void testOnFailureWithElasticsearchException() { + ElasticsearchException exception = Mockito.mock(ElasticsearchException.class); + Mockito.when(exception.status()).thenReturn(RestStatus.INTERNAL_SERVER_ERROR); + actionListener.set(RequestType.search, listenerId , originalActionListener); + actionListener.onFailure(exception); + String[] metricsValues = readMetricsValue(); + assertEquals("HTTPRespCode:500", metricsValues[2]); + assertTrue( metricsValues[3].contains("Exception:org.elasticsearch.ElasticsearchException")); + } @Test - public void testHttpMetrics() { - System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); - long startTimeInMills = 1553725339; - PerformanceAnalyzerActionListener performanceanalyzerActionListener = new PerformanceAnalyzerActionListener(); - performanceanalyzerActionListener.saveMetricValues("XYZADFAS", startTimeInMills, "bulk", "bulkId", "start"); - String fetchedValue = PerformanceAnalyzerMetrics.getMetric( - PluginSettings.instance().getMetricsLocation() - + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/http/bulk/bulkId/start"); - assertEquals("XYZADFAS", fetchedValue); - - String startMetricsValue = performanceanalyzerActionListener.generateStartMetrics(123, "val2", 0).toString(); - performanceanalyzerActionListener.saveMetricValues(startMetricsValue, startTimeInMills, "search", "searchId1", "start"); - fetchedValue = PerformanceAnalyzerMetrics.getMetric( - PluginSettings.instance().getMetricsLocation() - + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/http/search/searchId1/start"); - assertEquals(startMetricsValue, fetchedValue); - - String finishMetricsValue = performanceanalyzerActionListener.generateFinishMetrics(456, 200, "val4").toString(); - performanceanalyzerActionListener.saveMetricValues(finishMetricsValue, startTimeInMills, "search", "searchId1", "finish"); - fetchedValue = PerformanceAnalyzerMetrics.getMetric( - PluginSettings.instance().getMetricsLocation() - + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/http/search/searchId1/finish"); - assertEquals(finishMetricsValue, fetchedValue); - - performanceanalyzerActionListener.saveMetricValues(finishMetricsValue, startTimeInMills, "search", "searchId2", "finish"); - fetchedValue = PerformanceAnalyzerMetrics.getMetric( - PluginSettings.instance().getMetricsLocation() - + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/http/search/searchId2/finish"); - assertEquals(finishMetricsValue, fetchedValue); - - PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation() - + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)); + public void testOnFailureWithException() { + NullPointerException exception = new NullPointerException(); + actionListener.set(RequestType.search, listenerId , originalActionListener); + actionListener.onFailure(exception); + String[] metricsValues = readMetricsValue(); + assertEquals("HTTPRespCode:-1", metricsValues[2]); + assertTrue( metricsValues[3].contains("Exception:java.lang.NullPointerException")); + } + + private void testOnResponse(ActionResponse response) { + actionListener.onResponse(response); + + String[] metricsValues = readMetricsValue(); + assertEquals("HTTPRespCode:200", metricsValues[2]); + assertEquals("Exception:", metricsValues[3]); + } + + private String[] readMetricsValue() { + List metrics = TestUtil.readEvents(); + assert metrics.size() == 1; + String[] metricsValues = metrics.get(0).value.split("\n"); + assert metricsValues.length == 4; + return metricsValues; } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CacheConfigMetricsCollectorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CacheConfigMetricsCollectorTests.java index 8fa2fbee..b97b28b3 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CacheConfigMetricsCollectorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CacheConfigMetricsCollectorTests.java @@ -46,6 +46,9 @@ public void init() { MetricsConfiguration.CONFIG_MAP.put(CacheConfigMetricsCollector.class, MetricsConfiguration.cdefault); collector = new CacheConfigMetricsCollector(); + + //clean metricQueue before running every test + TestUtil.readEvents(); } @After diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java index 96a931d0..792ad583 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java @@ -47,6 +47,9 @@ public void init() { MetricsConfiguration.CONFIG_MAP.put(CircuitBreakerCollector.class, MetricsConfiguration.cdefault); collector = new CircuitBreakerCollector(); + + //clean metricQueue before running every test + TestUtil.readEvents(); } @After diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceEventMetricsTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceEventMetricsTests.java new file mode 100644 index 00000000..4d592e1b --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceEventMetricsTests.java @@ -0,0 +1,131 @@ +/* + * Copyright <2020> Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.Event; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.TestUtil; +import java.util.List; +import org.apache.commons.lang3.SystemUtils; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.service.SourcePrioritizedRunnable; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class MasterServiceEventMetricsTests { + private long startTimeInMills = 1153721339; + private MasterServiceEventMetrics masterServiceEventMetrics; + private ThreadPool threadPool; + + @BeforeClass + public static void setup() { + // this test only runs in Linux system + // as some of the static members of the ThreadList class are specific to Linux + org.junit.Assume.assumeTrue(SystemUtils.IS_OS_LINUX); + } + + @Before + public void init() { + threadPool = new TestThreadPool("test"); + ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool); + ESResources.INSTANCE.setClusterService(clusterService); + + MetricsConfiguration.CONFIG_MAP.put(MasterServiceEventMetrics.class, MetricsConfiguration.cdefault); + masterServiceEventMetrics = new MasterServiceEventMetrics(); + + //clean metricQueue before running every test + TestUtil.readEvents(); + } + + @After + public void tearDown(){ + threadPool.shutdownNow(); + } + + @Test + public void testGetMetricsPath() { + String expectedPath = PluginSettings.instance().getMetricsLocation() + + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills) + "/" + + PerformanceAnalyzerMetrics.sThreadsPath + "/" + "thread123" + "/" + + PerformanceAnalyzerMetrics.sMasterTaskPath + "/" + "task123" +"/" + + PerformanceAnalyzerMetrics.FINISH_FILE_NAME; + String actualPath = masterServiceEventMetrics.getMetricsPath(startTimeInMills, "thread123", "task123", PerformanceAnalyzerMetrics.FINISH_FILE_NAME); + assertEquals(expectedPath, actualPath); + + try { + masterServiceEventMetrics.getMetricsPath(startTimeInMills, "thread123", "task123"); + fail("Negative scenario test: Should have been a RuntimeException"); + } catch (RuntimeException ex) { + //- expecting exception...2 values passed; 3 expected + } + } + + + @Test + public void testGenerateFinishMetrics() { + assertEquals(-1 , masterServiceEventMetrics.lastTaskInsertionOrder); + masterServiceEventMetrics.generateFinishMetrics(startTimeInMills); + + masterServiceEventMetrics.lastTaskInsertionOrder = 1; + masterServiceEventMetrics.generateFinishMetrics(startTimeInMills); + List metrics = TestUtil.readEvents(); + String[] jsonStrs = metrics.get(0).value.split("\n"); + assert jsonStrs.length == 2; + assertTrue(jsonStrs[1].contains(AllMetrics.MasterMetricValues.FINISH_TIME.toString())); + assertEquals(-1 , masterServiceEventMetrics.lastTaskInsertionOrder); + } + + @Test + public void testCollectMetrics() throws Exception { + PrioritizedEsThreadPoolExecutor prioritizedEsThreadPoolExecutor = (PrioritizedEsThreadPoolExecutor) masterServiceEventMetrics + .getMasterServiceTPExecutorField().get(ESResources.INSTANCE.getClusterService().getMasterService()); + SourcePrioritizedRunnable runnable = new SourcePrioritizedRunnable(Priority.HIGH, "_add_listener_") { + @Override + public void run() { + try { + Thread.sleep(100L); //dummy runnable + } catch (InterruptedException e) { + } + } + }; + + prioritizedEsThreadPoolExecutor.submit(runnable); + Thread.sleep(1L); // don't delete it + + masterServiceEventMetrics.collectMetrics(startTimeInMills); + List jsonStrs = TestUtil.readMetricsInJsonString(6); + assertTrue(jsonStrs.get(0).contains(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString())); + assertTrue(jsonStrs.get(1).contains(AllMetrics.MasterMetricValues.START_TIME.toString())); + assertTrue(jsonStrs.get(2).contains(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString())); + assertTrue(jsonStrs.get(3).contains(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString())); + assertTrue(jsonStrs.get(4).contains(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString())); + } +} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetricsTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetricsTests.java index 1feefac7..4e2541f9 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetricsTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetricsTests.java @@ -15,69 +15,111 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; -import org.junit.Ignore; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingValue; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceEventMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.Event; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.TestUtil; +import java.util.List; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; -@Ignore -public class MasterServiceMetricsTests extends CustomMetricsLocationTestBase { +public class MasterServiceMetricsTests { + private MasterServiceMetrics masterServiceMetrics; + private long startTimeInMills = 1153721339; + private ThreadPool threadPool; - @Test - public void testMasterServiceMetrics() { - MetricsConfiguration.CONFIG_MAP.put(MasterServiceMetrics.class, new MetricsConfiguration.MetricConfig(1000, 0, 0)); - MetricsConfiguration.CONFIG_MAP.put(MasterServiceEventMetrics.class, new MetricsConfiguration.MetricConfig(1000, 0, 0)); + @Mock + private ClusterService mockedClusterService; + + @Before + public void init() { + initMocks(this); System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); - long startTimeInMills = 1353723339; + threadPool = new TestThreadPool("test"); + ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool); + ESResources.INSTANCE.setClusterService(clusterService); - MasterServiceMetrics masterServiceMetrics = new MasterServiceMetrics(); - masterServiceMetrics.saveMetricValues("master_metrics_value", startTimeInMills, "current", "start"); + MetricsConfiguration.CONFIG_MAP.put(MasterServiceMetrics.class, MetricsConfiguration.cdefault); + masterServiceMetrics = new MasterServiceMetrics(); + //clean metricQueue before running every test + TestUtil.readEvents(); + } - String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PluginSettings.instance().getMetricsLocation() + - PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/pending_tasks/current/start/"); - PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation() - + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)); - assertEquals("master_metrics_value", fetchedValue); + @After + public void tearDown() { + threadPool.shutdownNow(); + } - try { - masterServiceMetrics.saveMetricValues("master_metrics_value", startTimeInMills, "current"); - assertTrue("Negative scenario test: Should have been a RuntimeException", true); - } catch (RuntimeException ex) { - //- expecting exception...only 1 values passed; 2 expected - } + @Test + public void testGetMetricsPath() { + String expectedPath = PluginSettings.instance().getMetricsLocation() + + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills) + "/" + + PerformanceAnalyzerMetrics.sPendingTasksPath + "/" + + "current" + "/" + + PerformanceAnalyzerMetrics.FINISH_FILE_NAME; + String actualPath = masterServiceMetrics.getMetricsPath(startTimeInMills, "current", PerformanceAnalyzerMetrics.FINISH_FILE_NAME); + assertEquals(expectedPath, actualPath); try { - masterServiceMetrics.saveMetricValues("master_metrics_value", startTimeInMills); - assertTrue("Negative scenario test: Should have been a RuntimeException", true); + masterServiceMetrics.getMetricsPath(startTimeInMills, "current"); + fail("Negative scenario test: Should have been a RuntimeException"); } catch (RuntimeException ex) { - //- expecting exception...only 0 values passed; 2 expected + //- expecting exception...1 values passed; 2 expected } + } - try { - masterServiceMetrics.saveMetricValues("master_metrics_value", startTimeInMills, "current", "start", "123"); - assertTrue("Negative scenario test: Should have been a RuntimeException", true); - } catch (RuntimeException ex) { - //- expecting exception...only 3 values passed; 2 expected - } + @Test + public void testCollectMetrics() { + masterServiceMetrics.collectMetrics(startTimeInMills); + String jsonStr = readMetricsInJsonString(1); + assertTrue(jsonStr.contains(MasterPendingValue.Constants.PENDING_TASKS_COUNT_VALUE)); + } - MasterServiceEventMetrics masterServiceEventMetrics = new MasterServiceEventMetrics(); - try { - masterServiceEventMetrics.getMasterServiceTPExecutorField(); - masterServiceEventMetrics.getPrioritizedTPExecutorCurrentField(); - masterServiceEventMetrics.getPrioritizedTPExecutorAddPendingMethod(); - masterServiceEventMetrics.getTPExecutorWorkersField(); - masterServiceEventMetrics.getWorkerThreadField(); - } catch (Exception exception) { - assertTrue("There shouldn't be any exception in the code; Please check the reflection code for any changes", true); + @Test + public void testWithMockClusterService() { + ESResources.INSTANCE.setClusterService(mockedClusterService); + masterServiceMetrics.collectMetrics(startTimeInMills); + String jsonStr = readMetricsInJsonString(0); + assertNull(jsonStr); + + ESResources.INSTANCE.setClusterService(mockedClusterService); + when(mockedClusterService.getMasterService()).thenThrow(new RuntimeException()); + masterServiceMetrics.collectMetrics(startTimeInMills); + jsonStr = readMetricsInJsonString(0); + assertNull(jsonStr); + + ESResources.INSTANCE.setClusterService(null); + masterServiceMetrics.collectMetrics(startTimeInMills); + jsonStr = readMetricsInJsonString(0); + assertNull(jsonStr); + } + + private String readMetricsInJsonString(int size) { + List metrics = TestUtil.readEvents(); + assert metrics.size() == size; + if (size != 0) { + String[] jsonStrs = metrics.get(0).value.split("\n"); + assert jsonStrs.length == 2; + return jsonStrs[1]; + } else { + return null; } } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeDetailsCollectorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeDetailsCollectorTests.java index f76c620c..fb6a1e1f 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeDetailsCollectorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeDetailsCollectorTests.java @@ -67,6 +67,9 @@ public void init() { MetricsConfiguration.CONFIG_MAP.put(NodeDetailsCollector.class, MetricsConfiguration.cdefault); collector = new NodeDetailsCollector(configOverrides); + + //clean metricQueue before running every test + TestUtil.readEvents(); } @After diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsAllShardsMetricsCollectorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsAllShardsMetricsCollectorTests.java index 8c84c3a9..51f1cbc7 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsAllShardsMetricsCollectorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsAllShardsMetricsCollectorTests.java @@ -36,7 +36,7 @@ public class NodeStatsAllShardsMetricsCollectorTests extends ESSingleNodeTestCase { private static final String TEST_INDEX = "test"; private NodeStatsAllShardsMetricsCollector nodeStatsAllShardsMetricsCollector; - private long startTimeInMills = 1153721339;; + private long startTimeInMills = 1153721339; @Before public void init() { @@ -45,6 +45,9 @@ public void init() { MetricsConfiguration.CONFIG_MAP.put(NodeStatsAllShardsMetricsCollector.class, MetricsConfiguration.cdefault); nodeStatsAllShardsMetricsCollector = new NodeStatsAllShardsMetricsCollector(null); + + //clean metricQueue before running every test + TestUtil.readEvents(); } @After diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollectorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollectorTests.java index 599df6f0..271fbeac 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollectorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollectorTests.java @@ -51,6 +51,9 @@ public void init() { MetricsConfiguration.CONFIG_MAP.put(NodeStatsAllShardsMetricsCollector.class, MetricsConfiguration.cdefault); collector = new NodeStatsFixedShardsMetricsCollector(controller); + + //clean metricQueue before running every test + TestUtil.readEvents(); } @After diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTests.java index 38421f1b..380f58af 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ShardStateCollectorTests.java @@ -70,6 +70,9 @@ public void init() { controller = Mockito.mock(PerformanceAnalyzerController.class); configOverrides = Mockito.mock(ConfigOverridesWrapper.class); shardStateCollector = new ShardStateCollector(controller, configOverrides); + + //clean metricQueue before running every test + TestUtil.readEvents(); } @Test diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerControllerTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerControllerTests.java new file mode 100644 index 00000000..fa26a4ab --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerControllerTests.java @@ -0,0 +1,39 @@ +package com.amazon.opendistro.elasticsearch.performanceanalyzer.config; + +import static com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PerformanceAnalyzerController.DEFAULT_NUM_OF_SHARDS_PER_COLLECTION; +import static org.junit.Assert.assertEquals; +import static org.mockito.MockitoAnnotations.initMocks; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ScheduledMetricCollectorsExecutor; +import java.nio.file.Paths; +import org.elasticsearch.common.settings.Settings; +import org.junit.Before; +import org.junit.Test; + +public class PerformanceAnalyzerControllerTests { + private static final int NUM_OF_SHARDS_PER_COLLECTION = 1; + private Settings settings; + private PerformanceAnalyzerController controller; + + + @Before + public void init() { + initMocks(this); + settings = Settings.builder().put("path.home", "./").build(); + ESResources.INSTANCE.setSettings(settings); + ESResources.INSTANCE.setConfigPath(Paths.get("build/tmp/junit_metrics")); + controller = new PerformanceAnalyzerController(new ScheduledMetricCollectorsExecutor()); + + } + + @Test + public void testGetNodeStatsShardsPerCollection() { + assertEquals(DEFAULT_NUM_OF_SHARDS_PER_COLLECTION, controller.getNodeStatsShardsPerCollection()); + + controller.updateNodeStatsShardsPerCollection(NUM_OF_SHARDS_PER_COLLECTION); + assertEquals(NUM_OF_SHARDS_PER_COLLECTION, controller.getNodeStatsShardsPerCollection()); + + + } +} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/handler/NodeStatsSettingHandlerTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/handler/NodeStatsSettingHandlerTests.java new file mode 100644 index 00000000..c433d10d --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/handler/NodeStatsSettingHandlerTests.java @@ -0,0 +1,51 @@ +/* + * Copyright <2020> Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.handler; + +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PerformanceAnalyzerController; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.ClusterSettingsManager; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +public class NodeStatsSettingHandlerTests { + private NodeStatsSettingHandler handler; + + @Mock private PerformanceAnalyzerController controller; + @Mock private ClusterSettingsManager clusterSettingsManager; + + @Before + public void init() { + initMocks(this); + handler = new NodeStatsSettingHandler(controller, clusterSettingsManager); + } + + @Test + public void testOnSettingUpdate() { + Integer newSettingValue = null; + handler.onSettingUpdate(newSettingValue); + verify(controller, never()).updateNodeStatsShardsPerCollection(anyInt()); + + newSettingValue = 1; + handler.onSettingUpdate(newSettingValue); + verify(controller).updateNodeStatsShardsPerCollection(newSettingValue); + } +} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerClusterSettingHandlerTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/handler/PerformanceAnalyzerClusterSettingHandlerTests.java similarity index 98% rename from src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerClusterSettingHandlerTests.java rename to src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/handler/PerformanceAnalyzerClusterSettingHandlerTests.java index 18a3c2e2..72efe28c 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PerformanceAnalyzerClusterSettingHandlerTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/setting/handler/PerformanceAnalyzerClusterSettingHandlerTests.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistro.elasticsearch.performanceanalyzer.config; +package com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.handler; import static org.junit.Assert.assertEquals; import static org.mockito.MockitoAnnotations.initMocks; import static org.powermock.api.mockito.PowerMockito.when; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PerformanceAnalyzerController; import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.ClusterSettingsManager; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.setting.handler.PerformanceAnalyzerClusterSettingHandler; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/whoami/WhoAmITests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/whoami/WhoAmITests.java new file mode 100644 index 00000000..ec2d66fe --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/whoami/WhoAmITests.java @@ -0,0 +1,79 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.whoami; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; +import java.io.IOException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +@SuppressWarnings("unchecked") +public class WhoAmITests { + private TransportWhoAmIAction transportWhoAmIAction; + private WhoAmIAction whoAmIAction; + private WhoAmIResponse response; + + @Mock private IndicesService indicesService; + @Mock private ActionListener listener; + @Mock private TransportService transportService; + @Mock private ActionFilters actionFilters; + @Mock private Task task; + + @Before + public void init() { + initMocks(this); + + transportWhoAmIAction = new TransportWhoAmIAction(transportService, actionFilters, indicesService); + whoAmIAction = WhoAmIAction.INSTANCE; + response = new WhoAmIResponse(); + } + + @Test + public void testDoExecute() { + WhoAmIRequestBuilder builder = new WhoAmIRequestBuilder(null); + transportWhoAmIAction.doExecute(task, new WhoAmIRequest(), listener); + verify(listener).onResponse(any()); + assertEquals(indicesService, ESResources.INSTANCE.getIndicesService()); + } + + @Test + public void testWhoAmIActionReader() { + Reader reader = whoAmIAction.getResponseReader(); + assertEquals(WhoAmIAction.responseReader, reader); + } + + @Test + public void testWhoAmIActionResponse() throws IOException { + XContentBuilder contentBuilder = XContentBuilder.builder(XContentType.JSON.xContent()); + XContentBuilder builder = response.toXContent(contentBuilder, null); + response.writeTo(null); + assertEquals(contentBuilder, builder); + } +} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListenerTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListenerTests.java new file mode 100644 index 00000000..80d25ada --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListenerTests.java @@ -0,0 +1,193 @@ +/* + * Copyright <2020> Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.listener; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.MockitoAnnotations.initMocks; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PerformanceAnalyzerController; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm.ThreadList; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.TestUtil; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.commons.lang3.SystemUtils; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.internal.ShardSearchRequest; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; + +public class PerformanceAnalyzerSearchListenerTests { + private static final long TOOK_IN_NANOS = 10; + private static final String EXCEPTION = StatExceptionCode.ES_REQUEST_INTERCEPTOR_ERROR.toString(); + + private PerformanceAnalyzerSearchListener searchListener; + private StatsCollector statsCollector; + private long startTimeInMills = 1253721339; + private final AtomicInteger errorCount = new AtomicInteger(0); + + @Mock private SearchContext searchContext; + @Mock private ShardSearchRequest shardSearchRequest; + @Mock private ShardId shardId; + @Mock private PerformanceAnalyzerController controller; + + @BeforeClass + public static void setup() { + // this test only runs in Linux system + // as some of the static members of the ThreadList class are specific to Linux + org.junit.Assume.assumeTrue(SystemUtils.IS_OS_LINUX); + } + + @Before + public void init() { + initMocks(this); + Mockito.when(controller.isPerformanceAnalyzerEnabled()).thenReturn(true); + + MetricsConfiguration.CONFIG_MAP.put(ThreadList.class, MetricsConfiguration.cdefault); + searchListener = new PerformanceAnalyzerSearchListener(controller); + assertEquals(PerformanceAnalyzerSearchListener.class.getSimpleName(), searchListener.toString()); + + statsCollector = StatsCollector.instance(); + } + + @Test + public void testGetMetricsPath() { + String expectedPath = PluginSettings.instance().getMetricsLocation() + + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills) + "/" + + PerformanceAnalyzerMetrics.sThreadsPath + "/" + "SearchThread" + "/" + + "ShardQuery" +"/" + + "ShardSearchID" + "/" + + PerformanceAnalyzerMetrics.FINISH_FILE_NAME; + String actualPath = searchListener.getMetricsPath(startTimeInMills, "SearchThread", "ShardQuery", "ShardSearchID", PerformanceAnalyzerMetrics.FINISH_FILE_NAME); + assertEquals(expectedPath, actualPath); + + try { + searchListener.getMetricsPath(startTimeInMills, "SearchThread", "ShardQuery", "ShardSearchID"); + fail("Negative scenario test: Should have been a RuntimeException"); + } catch (RuntimeException ex) { + //- expecting exception...3 values passed; 4 expected + } + } + + @Test + public void testOnPreQueryPhase() { + initializeValidSearchContext(true); + searchListener.onPreQueryPhase(searchContext); + List jsonStrs = TestUtil.readMetricsInJsonString(4); + assertTrue(jsonStrs.get(0).contains(AllMetrics.CommonMetric.START_TIME.toString())); + assertTrue(jsonStrs.get(1).contains(AllMetrics.CommonDimension.INDEX_NAME.toString())); + assertTrue(jsonStrs.get(2).contains(AllMetrics.CommonDimension.SHARD_ID.toString())); + } + + @Test + public void testOnQueryPhase() { + initializeValidSearchContext(true); + searchListener.onQueryPhase(searchContext, TOOK_IN_NANOS); + List jsonStrs = TestUtil.readMetricsInJsonString(5); + assertTrue(jsonStrs.get(0).contains(AllMetrics.CommonMetric.FINISH_TIME.toString())); + assertTrue(jsonStrs.get(1).contains(AllMetrics.CommonDimension.FAILED.toString())); + assertTrue(jsonStrs.get(1).contains("false")); + assertTrue(jsonStrs.get(2).contains(AllMetrics.CommonDimension.INDEX_NAME.toString())); + assertTrue(jsonStrs.get(3).contains(AllMetrics.CommonDimension.SHARD_ID.toString())); + } + + + @Test + public void testOnFailedQueryPhase() { + initializeValidSearchContext(true); + searchListener.onFailedQueryPhase(searchContext); + List jsonStrs = TestUtil.readMetricsInJsonString(5); + assertTrue(jsonStrs.get(0).contains(AllMetrics.CommonMetric.FINISH_TIME.toString())); + assertTrue(jsonStrs.get(1).contains(AllMetrics.CommonDimension.FAILED.toString())); + assertTrue(jsonStrs.get(1).contains("true")); + assertTrue(jsonStrs.get(2).contains(AllMetrics.CommonDimension.INDEX_NAME.toString())); + assertTrue(jsonStrs.get(3).contains(AllMetrics.CommonDimension.SHARD_ID.toString())); + } + + @Test + public void testOnPreFetchPhase() { + initializeValidSearchContext(true); + searchListener.onPreFetchPhase(searchContext); + List jsonStrs = TestUtil.readMetricsInJsonString(4); + assertTrue(jsonStrs.get(0).contains(AllMetrics.CommonMetric.START_TIME.toString())); + assertTrue(jsonStrs.get(1).contains(AllMetrics.CommonDimension.INDEX_NAME.toString())); + assertTrue(jsonStrs.get(2).contains(AllMetrics.CommonDimension.SHARD_ID.toString())); + } + + @Test + public void testOnFetchPhase() { + initializeValidSearchContext(true); + searchListener.onFetchPhase(searchContext, TOOK_IN_NANOS); + List jsonStrs = TestUtil.readMetricsInJsonString(5); + assertTrue(jsonStrs.get(0).contains(AllMetrics.CommonMetric.FINISH_TIME.toString())); + assertTrue(jsonStrs.get(1).contains(AllMetrics.CommonDimension.FAILED.toString())); + assertTrue(jsonStrs.get(1).contains("false")); + assertTrue(jsonStrs.get(2).contains(AllMetrics.CommonDimension.INDEX_NAME.toString())); + assertTrue(jsonStrs.get(3).contains(AllMetrics.CommonDimension.SHARD_ID.toString())); + } + + @Test + public void testOnFailedFetchPhase() { + initializeValidSearchContext(true); + searchListener.onFailedFetchPhase(searchContext); + List jsonStrs = TestUtil.readMetricsInJsonString(5); + assertTrue(jsonStrs.get(0).contains(AllMetrics.CommonMetric.FINISH_TIME.toString())); + assertTrue(jsonStrs.get(1).contains(AllMetrics.CommonDimension.FAILED.toString())); + assertTrue(jsonStrs.get(1).contains("true")); + assertTrue(jsonStrs.get(2).contains(AllMetrics.CommonDimension.INDEX_NAME.toString())); + assertTrue(jsonStrs.get(3).contains(AllMetrics.CommonDimension.SHARD_ID.toString())); + } + + @Test + public void testInvalidSearchContext() { + initializeValidSearchContext(false); + + searchListener.onFailedFetchPhase(searchContext); + assertEquals(errorCount.incrementAndGet(), statsCollector.getCounters().get(EXCEPTION).intValue()); + searchListener.onPreFetchPhase(searchContext); + assertEquals(errorCount.incrementAndGet(), statsCollector.getCounters().get(EXCEPTION).intValue()); + searchListener.onFetchPhase(searchContext, TOOK_IN_NANOS); + assertEquals(errorCount.incrementAndGet(), statsCollector.getCounters().get(EXCEPTION).intValue()); + searchListener.onPreQueryPhase(searchContext); + assertEquals(errorCount.incrementAndGet(), statsCollector.getCounters().get(EXCEPTION).intValue()); + searchListener.onFailedQueryPhase(searchContext); + assertEquals(errorCount.incrementAndGet(), statsCollector.getCounters().get(EXCEPTION).intValue()); + searchListener.onQueryPhase(searchContext, TOOK_IN_NANOS); + assertEquals(errorCount.incrementAndGet(), statsCollector.getCounters().get(EXCEPTION).intValue()); + } + + private void initializeValidSearchContext(boolean isValid) { + if (isValid) { + Mockito.when(searchContext.request()).thenReturn(shardSearchRequest); + Mockito.when(shardSearchRequest.shardId()).thenReturn(shardId); + Mockito.when(shardId.getIndexName()).thenReturn("shardIndex"); + Mockito.when(shardId.getId()).thenReturn(1); + } else { + Mockito.when(searchContext.request()).thenReturn(null); + } + } +} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/AbstractReaderTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/AbstractReaderTests.java deleted file mode 100644 index 50f0d112..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/AbstractReaderTests.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.AbstractTests; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.DiskMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.HeapMetricsCollector.HeapStatus; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceMetrics.MasterPendingStatus; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NodeDetailsCollector.NodeDetailsStatus; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NodeStatsAllShardsMetricsCollector.NodeStatsMetricsAllShardsPerCollectionStatus; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NodeStatsFixedShardsMetricsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.GCType; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.NodeRole; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader.MetricPropertiesTests.FailureCondition; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import org.jooq.Condition; -import org.jooq.impl.DSL; -import org.junit.Ignore; - -@Ignore -public class AbstractReaderTests extends AbstractTests { - protected final String DB_URL; - - protected final Connection conn; - - public AbstractReaderTests() throws SQLException, ClassNotFoundException { - // make sure the sqlite classes and driver are loaded - Class.forName("org.sqlite.JDBC"); - DB_URL = "jdbc:sqlite:"; - System.setProperty("java.io.tmpdir", "/tmp"); - conn = DriverManager.getConnection(DB_URL); - } - - protected Condition getDimensionEqCondition(MetricDimension dimentionHeader, - String dimensionName) { - return DSL.field(dimentionHeader.toString(), String.class) - .eq(dimensionName); - } - - protected String createRelativePath(String... paths) { - StringBuilder sb = new StringBuilder(); - for (String path : paths) { - sb.append(path); - sb.append(File.separator); - } - return sb.toString(); - } - - protected void write(File f, boolean append, String... input) - throws IOException { - try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter( - new FileOutputStream(f, append)))) { - for (String line : input) { - writer.append(line); - writer.newLine(); - } - } catch (IOException e1) { - e1.printStackTrace(); - } - } - - protected String getCurrentMilliSeconds(long currentTimeMillis) { - return new StringBuilder().append("{\"") - .append(PerformanceAnalyzerMetrics.METRIC_CURRENT_TIME).append("\"") - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(currentTimeMillis).append("}").toString(); - } - - protected String createDiskMetrics(String name, double utilization, - double await, double serviceRate) { - StringBuffer value = new StringBuffer(); - - value.append(new DiskMetrics(name, utilization, await, serviceRate) - .serialize()); - - return value.toString(); - } - - protected String createPendingTaskMetrics(int pendingTaskCount) { - StringBuffer value = new StringBuffer(); - - value.append(new MasterPendingStatus(pendingTaskCount) - .serialize()); - - return value.toString(); - } - - protected String createShardStatMetrics(long indexingThrottleTime, - long queryCacheHitCount, long queryCacheMissCount, - long queryCacheInBytes, long fieldDataEvictions, - long fieldDataInBytes, long requestCacheHitCount, - long requestCacheMissCount, long requestCacheEvictions, - long requestCacheInBytes, long refreshCount, long refreshTime, - long flushCount, long flushTime, long mergeCount, - long mergeTime, long mergeCurrent, long indexBufferBytes, - long segmentCount, long segmentsMemory, long termsMemory, - long storedFieldsMemory, long termVectorsMemory, - long normsMemory, long pointsMemory, long docValuesMemory, - long indexWriterMemory, long versionMapMemory, - long bitsetMemory, long shardSizeInBytes, FailureCondition condition) { - // dummyCollector is only used to create the json string - NodeStatsFixedShardsMetricsCollector dummyCollectorFewShards = new NodeStatsFixedShardsMetricsCollector(null); - String str = (dummyCollectorFewShards.new NodeStatsMetricsFixedShardsPerCollectionStatus( - indexingThrottleTime, - refreshCount, - refreshTime, - flushCount, - flushTime, - mergeCount, - mergeTime, - mergeCurrent, - indexBufferBytes, - segmentCount, - segmentsMemory, - termsMemory, - storedFieldsMemory, - termVectorsMemory, - normsMemory, - pointsMemory, - docValuesMemory, - indexWriterMemory, - versionMapMemory, - bitsetMemory, shardSizeInBytes)).serialize(); - - str += (new NodeStatsMetricsAllShardsPerCollectionStatus( - queryCacheHitCount, - queryCacheMissCount, - queryCacheInBytes, - fieldDataEvictions, - fieldDataInBytes, - requestCacheHitCount, - requestCacheMissCount, - requestCacheEvictions, - requestCacheInBytes)).serialize(); - - if (condition == FailureCondition.INVALID_JSON_METRIC) { - str = str.substring(1); - } - return str; - } - - protected String createHeapMetrics(GCType name, long committed, long init, - long max, long used) { - return new HeapStatus(name.toString(), committed, init, max, used) - .serialize(); - } - - protected String createHeapMetrics(GCType name, long collectionCount, - long collectionTime) { - return new HeapStatus(name.toString(), - collectionCount, - collectionTime).serialize(); - } - - protected String createNodeDetailsMetrics(String id, String ipAddress) { - StringBuffer value = new StringBuffer(); - - value.append(new NodeDetailsStatus(id, ipAddress, NodeRole.UNKNOWN.toString(), false) - .serialize()); - - return value.toString(); - } - - static void setFinalStatic(Field field, Object newValue) throws Exception { - field.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - field.set(null, newValue); - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesTests.java deleted file mode 100644 index 9bfa373e..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesTests.java +++ /dev/null @@ -1,414 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import java.io.File; -import java.io.IOException; -import java.sql.SQLException; - -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.impl.DSL; -import org.junit.Ignore; -import org.junit.Test; -import org.mockito.Mockito; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.GCType; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HeapDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HeapValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricName; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardStatsDerivedDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardStatsValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@Ignore -public class MetricPropertiesTests extends AbstractReaderTests { - - public MetricPropertiesTests() throws SQLException, ClassNotFoundException { - super(); - } - - @Test - public void testHeap() throws Exception { - long currTimestamp = System.currentTimeMillis() + 4000; - - long currTimeBucket = PerformanceAnalyzerMetrics.getTimeInterval(currTimestamp); - String currentTimeBucketStr = String.valueOf(currTimeBucket); - temporaryFolder.newFolder(currentTimeBucketStr); - File output = temporaryFolder.newFile(createRelativePath( - currentTimeBucketStr, PerformanceAnalyzerMetrics.sHeapPath)); - - write(output, false, - PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds(), - createHeapMetrics(GCType.TOT_YOUNG_GC, 0, 0), - createHeapMetrics(GCType.TOT_FULL_GC, 0, 0), - createHeapMetrics(GCType.SURVIVOR, 69730304, 69730304, 69730304, - 6343808), - createHeapMetrics(GCType.PERM_GEN, 144998400, 0, -1, 137707592), - createHeapMetrics(GCType.OLD_GEN, 32051232768L, 32051232768L, - 32051232768L, 650876744), - createHeapMetrics(GCType.EDEN, 558432256, 558432256, 558432256, - 367013104), - createHeapMetrics(GCType.NON_HEAP, 259436544, 2555904, -1, - 249276784), - createHeapMetrics(GCType.HEAP, 32679395328L, 32749125632L, - 32679395328L, 1024254960L)); - - long lastSnapTimestamp = System.currentTimeMillis() - 1000; - - MetricProperties heapProperty = MetricPropertiesConfig.getInstance() - .getProperty(MetricName.HEAP_METRICS); - - heapProperty.getHandler().setRootLocation( - temporaryFolder.getRoot().getCanonicalPath() + File.separator); - - MemoryDBSnapshot heapSnap = new MemoryDBSnapshot(this.conn, - MetricName.HEAP_METRICS, currTimestamp); - - boolean res = heapProperty.dispatch(heapSnap, - currTimestamp, lastSnapTimestamp); - - assertTrue(res); - - assertTrue (GCType.values().length == heapSnap.fetchAll().size()); - - @SuppressWarnings("unchecked") - Field[] fields = new Field[6]; - - fields[0] = DSL.field(HeapValue.GC_COLLECTION_EVENT.toString(), - Double.class); - fields[1] = - DSL.field(HeapValue.GC_COLLECTION_TIME.toString(), Double.class); - fields[2] = DSL.field(HeapValue.HEAP_COMMITTED.toString(), - Double.class); - fields[3] = DSL.field(HeapValue.HEAP_INIT.toString(), - Double.class); - fields[4] = DSL.field(HeapValue.HEAP_MAX.toString(), - Double.class); - fields[5] = DSL.field(HeapValue.HEAP_USED.toString(), - Double.class); - - Result resRecord = heapSnap.fetchMetric( - getDimensionEqCondition( - HeapDimension.MEM_TYPE, - GCType.TOT_YOUNG_GC.toString()), - fields); - - Record record0 = resRecord.get(0); - Double collectionCount = Double.parseDouble( - record0.get(fields[0]).toString()); - assertEquals(0, collectionCount, 0.001); - - Double collectionTime = Double.parseDouble( - record0.get(fields[1]).toString()); - assertEquals(0, collectionTime, 0.001); - - for (int i=2; i<6; i++) { - assertEquals(-2, record0.get(fields[i]), 0.001); - } - } - - private String createShardStatMetrics(long indexingThrottleTime, - long queryCacheHitCount, long queryCacheMissCount, - long queryCacheInBytes, long fieldDataEvictions, - long fieldDataInBytes, long requestCacheHitCount, - long requestCacheMissCount, long requestCacheEvictions, - long requestCacheInBytes, long refreshCount, long refreshTime, - long flushCount, long flushTime, long mergeCount, - long mergeTime, long mergeCurrent, long indexBufferBytes, - long segmentCount, long segmentsMemory, long termsMemory, - long storedFieldsMemory, long termVectorsMemory, - long normsMemory, long pointsMemory, long docValuesMemory, - long indexWriterMemory, long versionMapMemory, - long bitsetMemory, long shardSizeInBytes) { - return createShardStatMetrics( - indexingThrottleTime, - queryCacheHitCount, queryCacheMissCount, - queryCacheInBytes, fieldDataEvictions, - fieldDataInBytes, requestCacheHitCount, - requestCacheMissCount, requestCacheEvictions, - requestCacheInBytes, refreshCount, refreshTime, - flushCount, flushTime, mergeCount, - mergeTime, mergeCurrent, indexBufferBytes, - segmentCount, segmentsMemory, termsMemory, - storedFieldsMemory, termVectorsMemory, - normsMemory, pointsMemory, docValuesMemory, - indexWriterMemory, versionMapMemory, - bitsetMemory, shardSizeInBytes, - FailureCondition.NONE); - } - - @Test - public void testShardStat() throws Exception { - long currTimestamp = System.currentTimeMillis() + 4000; - - long currTimeBucket = PerformanceAnalyzerMetrics.getTimeInterval(currTimestamp); - String currentTimeBucketStr = String.valueOf(currTimeBucket); - - String moviesIndex = "movies-2013"; - String taxisIndex = "nyc_taxis"; - temporaryFolder.newFolder(currentTimeBucketStr, - PerformanceAnalyzerMetrics.sIndicesPath, moviesIndex); - temporaryFolder.newFolder(currentTimeBucketStr, - PerformanceAnalyzerMetrics.sIndicesPath, taxisIndex); - String indicesRelativePath = createRelativePath(currentTimeBucketStr, - PerformanceAnalyzerMetrics.sIndicesPath); - String moviesIndexeRelativePath = createRelativePath( - indicesRelativePath, moviesIndex); - String taxisIndexeRelativePath = createRelativePath(indicesRelativePath, - taxisIndex); - - File moviesShard0 = temporaryFolder - .newFile(createRelativePath(moviesIndexeRelativePath, "0")); - File moviesShard1 = temporaryFolder - .newFile(createRelativePath(moviesIndexeRelativePath, "1")); - - write(moviesShard0, false, PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds(), - createShardStatMetrics(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, - 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 2L, 8145L, 6565L, - 672L, 0L, 384L, 28L, 496L, 0L, 0L, 0L, 0L)); - - write(moviesShard1, false, PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds(), - createShardStatMetrics(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, - 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 2L, 8019L, 6445L, - 664L, 0L, 384L, 6L, 520L, 0L, 0L, 0L, 0L)); - - File taxisShard0 = temporaryFolder - .newFile(createRelativePath(taxisIndexeRelativePath, "0")); - - write(taxisShard0, false, PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds(), - createShardStatMetrics(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, - 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, - 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L)); - - long lastSnapTimestamp = System.currentTimeMillis() - 1000; - - MetricProperties shardStatProperty = MetricPropertiesConfig.getInstance() - .getProperty(MetricName.SHARD_STATS); - MetricPropertiesConfig.ShardStatFileHandler handler = new MetricPropertiesConfig.ShardStatFileHandler(); - MetricPropertiesConfig.ShardStatFileHandler spyHandler = Mockito.spy(handler); - shardStatProperty.setHandler(spyHandler); - - Mockito.doReturn(temporaryFolder.getRoot().getCanonicalPath()) - .when(spyHandler).getRootLocation(); - - MemoryDBSnapshot shardStatSnap = new MemoryDBSnapshot(this.conn, - MetricName.SHARD_STATS, currTimestamp); - - boolean res = shardStatProperty.dispatch(shardStatSnap, - currTimestamp, lastSnapTimestamp); - - assertTrue(res); - - assertTrue (3 == shardStatSnap.fetchAll().size()); - - @SuppressWarnings("unchecked") - Field[] fields = new Field[6]; - - fields[0] = DSL.field(ShardStatsValue.SEGMENTS_TOTAL.toString(), - Double.class); - fields[1] = DSL.field(ShardStatsValue.SEGMENTS_MEMORY.toString(), - Double.class); - fields[2] = DSL.field(ShardStatsValue.TERM_VECTOR_MEMORY.toString(), - Double.class); - fields[3] = DSL.field(ShardStatsValue.CACHE_FIELDDATA_EVICTION.toString(), - Double.class); - fields[4] = DSL.field(ShardStatsValue.DOC_VALUES_MEMORY.toString(), - Double.class); - fields[5] = DSL.field(ShardStatsValue.MERGE_EVENT.toString(), - Double.class); - - Result resRecord = shardStatSnap.fetchMetric( - getDimensionEqCondition( - ShardStatsDerivedDimension.INDEX_NAME, - moviesIndex), - fields); - - assertTrue(2 == resRecord.size()); - - Result resRecord2 = shardStatSnap.fetchMetric( - getDimensionEqCondition(ShardStatsDerivedDimension.INDEX_NAME, - moviesIndex) - .and(getDimensionEqCondition( - ShardStatsDerivedDimension.SHARD_ID, - "0")), - fields); - - Record record0 = resRecord2.get(0); - Double segmentCount = Double.parseDouble( - record0.get(fields[0]).toString()); - assertEquals(2, segmentCount, 0.001); - - Double segmentMemory = Double.parseDouble( - record0.get(fields[1]).toString()); - assertEquals(8145, segmentMemory, 0.001); - - Double termVectorMemory = Double.parseDouble( - record0.get(fields[2]).toString()); - assertEquals(0, termVectorMemory, 0.001); - - Double fieldDataEviction = Double.parseDouble( - record0.get(fields[3]).toString()); - assertEquals(0, fieldDataEviction, 0.001); - - Double docValuesMemory = Double.parseDouble( - record0.get(fields[4]).toString()); - assertEquals(496, docValuesMemory, 0.001); - - Double mergeCount = Double.parseDouble( - record0.get(fields[5]).toString()); - assertEquals(0, mergeCount, 0.001); - } - - enum FailureCondition { - BAD_FILE_NAME, EMPTY_FILE, MODIFIED_AFTER_START, ALREADY_PROCESSED, - INVALID_JSON_TIME, INVALID_JSON_METRIC, NONE - } - - public String getCurrentNonJsonMilliSeconds() { - return new StringBuilder() - .append(PerformanceAnalyzerMetrics.METRIC_CURRENT_TIME).append("\"") - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(System.currentTimeMillis()).append("}").toString(); - } - - private boolean createFailureScenario(FailureCondition condition) - throws Exception { - long currTimestamp = 0; - // we don't process files modified after currTimestamp - if (condition == FailureCondition.MODIFIED_AFTER_START) { - currTimestamp = System.currentTimeMillis() - 1000; - } else { - currTimestamp = System.currentTimeMillis() + 4000; - } - - long currTimeBucket = PerformanceAnalyzerMetrics.getTimeInterval(currTimestamp); - String currentTimeBucketStr = String.valueOf(currTimeBucket); - - String moviesIndex = "movies-2013"; - temporaryFolder.newFolder(currentTimeBucketStr, - PerformanceAnalyzerMetrics.sIndicesPath, moviesIndex); - - String indicesRelativePath = createRelativePath(currentTimeBucketStr, - PerformanceAnalyzerMetrics.sIndicesPath); - String moviesIndexeRelativePath = createRelativePath( - indicesRelativePath, moviesIndex); - - File moviesShard0 = null; - if (condition == FailureCondition.BAD_FILE_NAME) { - // this metric file is not numeric - moviesShard0 = temporaryFolder - .newFile(createRelativePath(moviesIndexeRelativePath, "X")); - } else { - moviesShard0 = temporaryFolder - .newFile(createRelativePath(moviesIndexeRelativePath, "0")); - } - - if (condition != FailureCondition.EMPTY_FILE) { - if (condition == FailureCondition.INVALID_JSON_TIME) { - write(moviesShard0, false, getCurrentNonJsonMilliSeconds(), - createShardStatMetrics(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, - 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 2L, - 8145L, 6565L, 672L, 0L, 384L, 28L, 496L, 0L, 0L, - 0L, 0L)); - } else if (condition == FailureCondition.INVALID_JSON_METRIC) { - write(moviesShard0, false, PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds(), - createShardStatMetrics(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, - 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 2L, - 8145L, 6565L, 672L, 0L, 384L, 28L, 496L, 0L, 0L, - 0L, 0L, condition)); - } - else { - write(moviesShard0, false, PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds(), - createShardStatMetrics(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, - 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 2L, - 8145L, 6565L, 672L, 0L, 384L, 28L, 496L, 0L, 0L, - 0L, 0L)); - } - } - - long lastSnapTimestamp = 0; - if (condition == FailureCondition.ALREADY_PROCESSED) { - // we don't process files if this file has already been processed - // previously - lastSnapTimestamp = System.currentTimeMillis(); - } else { - lastSnapTimestamp = System.currentTimeMillis() - 1000; - } - - - MetricProperties shardStatProperty = MetricPropertiesConfig - .getInstance().getProperty(MetricName.SHARD_STATS); - MetricPropertiesConfig.ShardStatFileHandler handler = new MetricPropertiesConfig.ShardStatFileHandler(); - MetricPropertiesConfig.ShardStatFileHandler spyHandler = Mockito.spy(handler); - shardStatProperty.setHandler(spyHandler); - - Mockito.doReturn(temporaryFolder.getRoot().getCanonicalPath()) - .when(spyHandler).getRootLocation(); - - MemoryDBSnapshot shardStatSnap = new MemoryDBSnapshot(this.conn, - MetricName.SHARD_STATS, currTimestamp); - - // no metrics parsed - return shardStatProperty.dispatch(shardStatSnap, - currTimestamp, lastSnapTimestamp); - } - - @Test(expected = IOException.class) - public void testBadPathPattern() throws Exception { - createFailureScenario(FailureCondition.BAD_FILE_NAME); - } - - @Test - public void testDefaultRootLocation() { - assertEquals( - PluginSettings.instance().getMetricsLocation(), - MetricPropertiesConfig - .createFileHandler(PerformanceAnalyzerMetrics.sCircuitBreakerPath) - .getRootLocation()); - } - - @Test - public void testEmptyFile() throws Exception { - assertTrue(!createFailureScenario(FailureCondition.EMPTY_FILE)); - } - - @Test - public void testModifiedAfterStart() throws Exception { - assertTrue( - !createFailureScenario(FailureCondition.MODIFIED_AFTER_START)); - } - - @Test - public void testAlreadyProcessed() throws Exception { - assertTrue(!createFailureScenario(FailureCondition.ALREADY_PROCESSED)); - } - - @Test - public void testInvalidJsonTime() throws Exception { - assertTrue(!createFailureScenario(FailureCondition.INVALID_JSON_TIME)); - } - - @Test - public void testInvalidJsonMetric() throws Exception { - assertTrue(!createFailureScenario(FailureCondition.INVALID_JSON_METRIC)); - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader_writer_shared/EventLogFileHandlerTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader_writer_shared/EventLogFileHandlerTests.java similarity index 90% rename from src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader_writer_shared/EventLogFileHandlerTest.java rename to src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader_writer_shared/EventLogFileHandlerTests.java index 78ccdcb3..9a1feb86 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader_writer_shared/EventLogFileHandlerTest.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader_writer_shared/EventLogFileHandlerTests.java @@ -1,22 +1,22 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared; -import java.io.File; - -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mock; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.HeapMetricsCollector; import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PerformanceAnalyzerController; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.config.PerformanceAnalyzerConfigAction; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.writer.EventLogQueueProcessor; -import static org.mockito.Mockito.when; -import static org.mockito.MockitoAnnotations.initMocks; +import java.io.File; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; -public class EventLogFileHandlerTest { - @Mock - private PerformanceAnalyzerController mockController; +public class EventLogFileHandlerTests { + @Mock private PerformanceAnalyzerController mockController; + @Mock private PerformanceAnalyzerConfigAction configAction; String pathToTestMetricsDir; @@ -27,6 +27,7 @@ public void init() { deleteDirectory(new File(pathToTestMetricsDir)); boolean newDir = new File(pathToTestMetricsDir).mkdir(); when(mockController.isPerformanceAnalyzerEnabled()).thenReturn(true); + PerformanceAnalyzerConfigAction.setInstance(configAction); } private boolean deleteDirectory(File directoryToBeDeleted) { diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/TestUtil.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/TestUtil.java index 8922bdef..d5b6ea0c 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/TestUtil.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/TestUtil.java @@ -3,6 +3,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.Event; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class TestUtil { @@ -11,4 +12,12 @@ public static List readEvents() { PerformanceAnalyzerMetrics.metricQueue.drainTo(metrics); return metrics; } + + public static List readMetricsInJsonString(int length) { + List metrics = readEvents(); + assert metrics.size() == 1; + String[] jsonStrs = metrics.get(0).value.split("\n"); + assert jsonStrs.length == length; + return Arrays.asList(jsonStrs).subList(1, jsonStrs.length); + } }