diff --git a/build.gradle b/build.gradle index ea0ce323..a1d4b4bb 100644 --- a/build.gradle +++ b/build.gradle @@ -21,6 +21,7 @@ buildscript { // This isn't applying from repositories.gradle so repeating it here repositories { mavenCentral() + mavenLocal() } dependencies { @@ -92,6 +93,7 @@ project.afterEvaluate { repositories { mavenCentral() + mavenLocal() } @@ -114,13 +116,12 @@ dependencies { all*.exclude group: "org.elasticsearch", module: "securemock" } - compile 'org.jooq:jooq:3.10.8' - compile 'org.bouncycastle:bcprov-jdk15on:1.60' - compile 'org.bouncycastle:bcpkix-jdk15on:1.60' - compile 'org.xerial:sqlite-jdbc:3.8.11.2' compile 'com.google.guava:guava:27.0.1-jre' + compile 'org.jooq:jooq:3.10.8' + compile 'com.amazon.opendistro.elasticsearch:performanceanalyzer:1.0'; compile 'com.fasterxml.jackson.core:jackson-annotations:2.8.11' compile 'com.fasterxml.jackson.core:jackson-databind:2.8.11' + compile 'com.fasterxml.jackson.core:jackson-databind:2.8.11' // JDK9+ has to run powermock 2+. https://github.com/powermock/powermock/issues/888 testCompile group: 'org.powermock', name: 'powermock-api-mockito2', version: '2.0.0' diff --git a/config/opendistro_performance_analyzer/log4j2.properties b/config/opendistro_performance_analyzer/log4j2.properties deleted file mode 100644 index fa53dde2..00000000 --- a/config/opendistro_performance_analyzer/log4j2.properties +++ /dev/null @@ -1,14 +0,0 @@ -appender.stats_log_rolling.type = RollingFile -appender.stats_log_rolling.name = stats_log_rolling -appender.stats_log_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}performance_analyzer_plugin_stats.log -appender.stats_log_rolling.layout.type = PatternLayout -appender.stats_log_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}performance_analyzer_plugin_stats-%d{yyyy-MM-dd}.log -appender.stats_log_rolling.policies.type = Policies -appender.stats_log_rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.stats_log_rolling.policies.time.interval = 1 -appender.stats_log_rolling.policies.time.modulate = true - -logger.stats_log.name = stats_log -logger.stats_log.level = info -logger.stats_log.appenderRef.stats_log_rolling.ref = stats_log_rolling -logger.stats_log.additivity = false diff --git a/licenses/bcpkix-jdk15on-1.60.jar.sha1 b/licenses/bcpkix-jdk15on-1.60.jar.sha1 deleted file mode 100644 index 2217a947..00000000 --- a/licenses/bcpkix-jdk15on-1.60.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d0c46320fbc07be3a24eb13a56cee4e3d38e0c75 \ No newline at end of file diff --git a/licenses/bcpkix-jdk15on-LICENSE.txt b/licenses/bcpkix-jdk15on-LICENSE.txt deleted file mode 100644 index b6a4e728..00000000 --- a/licenses/bcpkix-jdk15on-LICENSE.txt +++ /dev/null @@ -1,12 +0,0 @@ -Please note this should be read in the same way as the MIT license. - -Please also note this licensing model is made possible through funding from donations and the sale of support contracts. - -LICENSE -Copyright (c) 2000 - 2019 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses/bcprov-jdk15on-1.60.jar.sha1 b/licenses/bcprov-jdk15on-1.60.jar.sha1 deleted file mode 100644 index e0604c45..00000000 --- a/licenses/bcprov-jdk15on-1.60.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bd47ad3bd14b8e82595c7adaa143501e60842a84 \ No newline at end of file diff --git a/licenses/bcprov-jdk15on-LICENSE.txt b/licenses/bcprov-jdk15on-LICENSE.txt deleted file mode 100644 index b6a4e728..00000000 --- a/licenses/bcprov-jdk15on-LICENSE.txt +++ /dev/null @@ -1,12 +0,0 @@ -Please note this should be read in the same way as the MIT license. - -Please also note this licensing model is made possible through funding from donations and the sale of support contracts. - -LICENSE -Copyright (c) 2000 - 2019 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses/bcprov-jdk15on-NOTICE.txt b/licenses/bcprov-jdk15on-NOTICE.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/licenses/performanceanalyzer-1.0.jar.sha1 b/licenses/performanceanalyzer-1.0.jar.sha1 new file mode 100644 index 00000000..e6035317 --- /dev/null +++ b/licenses/performanceanalyzer-1.0.jar.sha1 @@ -0,0 +1 @@ +c1334d0a10bde9be7a31fd633afaa22bdfa9ed71 \ No newline at end of file diff --git a/licenses/performanceanalyzer-LICENSE.txt b/licenses/performanceanalyzer-LICENSE.txt new file mode 100644 index 00000000..2116d6d9 --- /dev/null +++ b/licenses/performanceanalyzer-LICENSE.txt @@ -0,0 +1,12 @@ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/licenses/bcpkix-jdk15on-NOTICE.txt b/licenses/performanceanalyzer-NOTICE.txt similarity index 100% rename from licenses/bcpkix-jdk15on-NOTICE.txt rename to licenses/performanceanalyzer-NOTICE.txt diff --git a/licenses/sqlite-jdbc-3.8.11.2.jar.sha1 b/licenses/sqlite-jdbc-3.8.11.2.jar.sha1 deleted file mode 100644 index 68c85690..00000000 --- a/licenses/sqlite-jdbc-3.8.11.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dec208cba8debb0a8b9204b08d8d887d63041f0f \ No newline at end of file diff --git a/licenses/sqlite-jdbc-LICENSE.txt b/licenses/sqlite-jdbc-LICENSE.txt deleted file mode 100644 index d6456956..00000000 --- a/licenses/sqlite-jdbc-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/licenses/sqlite-jdbc-NOTICE.txt b/licenses/sqlite-jdbc-NOTICE.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/pa_bin/performance-analyzer-agent b/pa_bin/performance-analyzer-agent index 63892ef6..356ae2fd 100755 --- a/pa_bin/performance-analyzer-agent +++ b/pa_bin/performance-analyzer-agent @@ -24,18 +24,18 @@ else fi if ! echo $* | grep -E '(^-d |-d$| -d |--daemonize$|--daemonize )' > /dev/null; then - exec $JAVA_HOME/bin/java -Des.path.home=$ES_HOME -Dlog4j.configurationFile=$ES_HOME/plugins/opendistro_performance_analyzer/pa_config/log4j2.xml \ + exec $JAVA_HOME/bin/java -Des.path.home=$ES_HOME -Dlog4j.configurationFile=$ES_HOME/opendistro_performance_analyzer/pa_config/log4j2.xml \ -DconfigFilePath=$3 \ -Xms64M -Xmx64M -XX:+UseSerialGC -XX:CICompilerCount=1 -XX:-TieredCompilation -XX:InitialCodeCacheSize=4096 \ -XX:InitialBootClassLoaderMetaspaceSize=30720 -XX:MaxRAM=400m \ - -cp $ES_HOME/lib/*:$ES_HOME/plugins/opendistro_performance_analyzer/* com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp + -cp $ES_HOME/lib/*:$ES_HOME/opendistro_performance_analyzer/* com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp else echo 'Starting deamon' - exec $JAVA_HOME/bin/java -Des.path.home=$ES_HOME -Dlog4j.configurationFile=$ES_HOME/plugins/opendistro_performance_analyzer/pa_config/log4j2.xml \ + exec $JAVA_HOME/bin/java -Des.path.home=$ES_HOME -Dlog4j.configurationFile=$ES_HOME/opendistro_performance_analyzer/pa_config/log4j2.xml \ -DconfigFilePath=$3 \ -Xms64M -Xmx64M -XX:+UseSerialGC -XX:CICompilerCount=1 -XX:-TieredCompilation -XX:InitialCodeCacheSize=4096 \ -XX:InitialBootClassLoaderMetaspaceSize=30720 -XX:MaxRAM=400m \ - -cp $ES_HOME/lib/*:$ES_HOME/plugins/opendistro_performance_analyzer/* com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp & + -cp $ES_HOME/lib/*:$ES_HOME/opendistro_performance_analyzer/* com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp & pid=$! PID_LOC=/tmp/performance-analyzer-agent diff --git a/pa_config/log4j2.xml b/pa_config/log4j2.xml index 692d595f..31386c9b 100644 --- a/pa_config/log4j2.xml +++ b/pa_config/log4j2.xml @@ -4,7 +4,7 @@ - + @@ -16,7 +16,7 @@ - + diff --git a/pa_config/supervisord.conf b/pa_config/supervisord.conf index 65792c10..8a7b4eda 100644 --- a/pa_config/supervisord.conf +++ b/pa_config/supervisord.conf @@ -28,5 +28,5 @@ serverurl=/usr/share/supervisord.sock files = /etc/supervisor/conf.d/*.conf [program:performance_analyzer] -command=/usr/share/elasticsearch/plugins/opendistro_performance_analyzer/pa_bin/performance-analyzer-agent /usr/share/elasticsearch +command=/usr/share/elasticsearch/performance-analyzer-rca-1.0.0-SNAPSHOT/bin/performance-analyzer-rca /usr/share/elasticsearch user=1000 diff --git a/release-notes b/release-notes index 185b86eb..f837d38c 100644 --- a/release-notes +++ b/release-notes @@ -1,3 +1,4 @@ +Backporting fixes from master ## Version 1.2.0 (Version compatible with elasticsearch 7.2.0) ## New Features This is the release of the Open Distro Performance Analyzer that will work with elasticsearch 7.2.0 diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/CertificateUtils.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/CertificateUtils.java deleted file mode 100644 index 1e9bf919..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/CertificateUtils.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.amazon.opendistro.elasticsearch.performanceanalyzer; - -import java.io.FileReader; -import java.security.KeyStore; -import java.security.PrivateKey; -import java.security.cert.Certificate; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; -import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; -import org.bouncycastle.cert.X509CertificateHolder; -import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.openssl.PEMParser; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; - -public class CertificateUtils { - - public static final String ALIAS_PRIVATE = "private"; - public static final String ALIAS_CERT = "cert"; - //The password is not used to encrypt keys on disk. - public static final String IN_MEMORY_PWD = "opendistro"; - private static final String CERTIFICATE_FILE_PATH = "certificate-file-path"; - private static final String PRIVATE_KEY_FILE_PATH = "private-key-file-path"; - private static final Logger LOGGER = LogManager.getLogger(CertificateUtils.class); - - public static Certificate getCertificate(final FileReader certReader) throws Exception { - try (PEMParser pemParser = new PEMParser(certReader)) { - X509CertificateHolder certificateHolder = (X509CertificateHolder) pemParser.readObject(); - Certificate caCertificate = new JcaX509CertificateConverter() - .setProvider("BC") - .getCertificate(certificateHolder); - return caCertificate; - } - } - - public static PrivateKey getPrivateKey(final FileReader keyReader) throws Exception { - try (PEMParser pemParser = new PEMParser(keyReader)) { - PrivateKeyInfo pki = (PrivateKeyInfo) pemParser.readObject(); - return BouncyCastleProvider.getPrivateKey(pki); - } - } - - public static KeyStore createKeyStore() throws Exception { - String certFilePath = PluginSettings.instance().getSettingValue(CERTIFICATE_FILE_PATH); - String keyFilePath = PluginSettings.instance().getSettingValue(PRIVATE_KEY_FILE_PATH); - PrivateKey pk = getPrivateKey(new FileReader(keyFilePath)); - KeyStore ks = createEmptyStore(); - Certificate certificate = getCertificate(new FileReader(certFilePath)); - ks.setCertificateEntry(ALIAS_CERT, certificate); - ks.setKeyEntry(ALIAS_PRIVATE, pk, IN_MEMORY_PWD.toCharArray(), new Certificate[]{certificate}); - return ks; - } - - public static KeyStore createEmptyStore() throws Exception { - KeyStore ks = KeyStore.getInstance("JKS"); - ks.load(null, IN_MEMORY_PWD.toCharArray()); - return ks; - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/ConfigStatus.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/ConfigStatus.java deleted file mode 100644 index b92f957e..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/ConfigStatus.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer; - -public final class ConfigStatus { - private boolean configMissingOrIncorrect = false; - public static final ConfigStatus INSTANCE = new ConfigStatus(); - - private ConfigStatus() { - } - - public boolean haveValidConfig() { - return !configMissingOrIncorrect; - } - - public void setConfigurationInvalid() { - configMissingOrIncorrect = true; - } - - -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/DBUtils.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/DBUtils.java deleted file mode 100644 index 5ef903d3..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/DBUtils.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer; - -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.SelectHavingStep; -import org.jooq.TableLike; -import org.jooq.impl.DSL; - -@SuppressWarnings("unchecked") -public class DBUtils { - public static boolean checkIfTableExists(DSLContext create, String tableName) { - Result res = create.select() - .from(DSL.table("sqlite_master")) - .where(DSL.field("type").eq("table").and( - DSL.field("name").eq(tableName))) - .fetch(); - return (res.size() > 0); - } - - public static Result fetchTables(DSLContext create) { - return create.select() - .from(DSL.table("sqlite_master")) - .where(DSL.field("type").eq("table")) - .fetch(); - } - - public static List> getStringFieldsFromList(Collection fieldNames) { - return fieldNames.stream() - .map(s -> DSL.field(DSL.name(s), String.class)) - .collect(Collectors.toList()); - } - - /** - * Same implementation as getStringFieldsFromList, but return a list - * allowing other kinds of fields other than String field. - * - * @param fieldNames a collection of field names - * - * @return a list of org.jooq.Field objects - * - */ - public static List> getFieldsFromList( - Collection fieldNames) { - return fieldNames.stream() - .map(s -> DSL.field(DSL.name(s), String.class)) - .collect(Collectors.toList()); - } - - public static List> getDoubleFieldsFromList(Collection fieldNames) { - return fieldNames.stream() - .map(s -> DSL.field(DSL.name(s), Double.class)) - .collect(Collectors.toList()); - } - - public static List> getStringFieldsFromTable(Collection fieldNames, - TableLike table) { - return fieldNames.stream() - .map(s -> table.field(s, String.class)) - .collect(Collectors.toList()); - } - - public static List> getSelectFieldsForMetricName(String metricName, List metrics, List dimensions) { - List> selectFields = DBUtils.getFieldsFromList(dimensions); - for (String metric : metrics) { - if (metric.equals(metricName)) { - selectFields.add(DSL.field(metric, Double.class).as(metric)); - } else { - selectFields.add(DSL.val(null, Double.class).as(metric)); - } - } - return selectFields; - } - - /** - * Get records by field and return as a set. - * @param table table select - * @param field field - * @param condition select condition - * @param create db connection - * @return records set - */ - public static Set getRecordSetByField(SelectHavingStep table, Field field, Condition condition, - final DSLContext create) { - Result records = create.select(field) - .from(table) - .where(condition) - .fetch(); - - Set res = new HashSet<>(); - for (int i = 0; i < records.size(); i++) { - res.add(records.get(i).get(0).toString()); - } - return res; - } - - public static String getAggFieldName(String fieldName, String aggName) { - return aggName + "_" + fieldName; - } - - public static Map> getDoubleFieldMapFromList( - Collection fieldNames) { - return fieldNames.stream().collect(Collectors.toMap(s -> s, - s -> DSL.field(DSL.name(s), Double.class))); - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/OSMetricsGeneratorFactory.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/OSMetricsGeneratorFactory.java deleted file mode 100644 index 378ef96d..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/OSMetricsGeneratorFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.OSMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux.LinuxOSMetricsGenerator; - -public class OSMetricsGeneratorFactory { - - private static final String OS_TYPE = System.getProperty("os.name"); - - public static OSMetricsGenerator getInstance() { - - if (isLinux()) { - return LinuxOSMetricsGenerator.getInstance(); - } else { - ConfigStatus.INSTANCE.setConfigurationInvalid(); - } - - return null; - } - - private static boolean isLinux() { - return OS_TYPE.toLowerCase().contains("linux"); - } - -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerApp.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerApp.java deleted file mode 100644 index c9221253..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerApp.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer; - -import java.io.File; - -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.security.KeyStore; -import java.util.concurrent.Executors; - - -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import java.security.cert.X509Certificate; -import java.security.Security; -import javax.net.ssl.X509TrustManager; -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLSession; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.TroubleshootingConfig; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader.ReaderMetricsProcessor; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rest.QueryMetricsRequestHandler; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.bouncycastle.jce.provider.BouncyCastleProvider; - -import com.sun.net.httpserver.HttpServer; -import com.sun.net.httpserver.HttpsServer; -import com.sun.net.httpserver.HttpsConfigurator; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ScheduledMetricCollectorsExecutor; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; - -public class PerformanceAnalyzerApp { - private static final int WEBSERVICE_DEFAULT_PORT = 9600; - private static final String WEBSERVICE_PORT_CONF_NAME = "webservice-listener-port"; - private static final String WEBSERVICE_BIND_HOST_NAME = "webservice-bind-host"; - //Use system default for max backlog. - private static final int INCOMING_QUEUE_LENGTH = 1; - public static final String QUERY_URL = "/_opendistro/_performanceanalyzer/metrics"; - private static final Logger LOG = LogManager.getLogger(PerformanceAnalyzerApp.class); - private static final ScheduledMetricCollectorsExecutor METRIC_COLLECTOR_EXECUTOR = new ScheduledMetricCollectorsExecutor(1, false); - - public static void main(String[] args) throws Exception { - ESResources.INSTANCE.setPluginFileLocation(System.getProperty("es.path.home") - + File.separator + "plugins" + File.separator + PerformanceAnalyzerPlugin.PLUGIN_NAME + File.separator); - - //Initialize settings before creating threads. - PluginSettings settings = PluginSettings.instance(); - - StatsCollector.STATS_TYPE = "agent-stats-metadata"; - METRIC_COLLECTOR_EXECUTOR.addScheduledMetricCollector(StatsCollector.instance()); - StatsCollector.instance().addDefaultExceptionCode(StatExceptionCode.READER_RESTART_PROCESSING); - METRIC_COLLECTOR_EXECUTOR.start(); - - Thread readerThread = new Thread(new Runnable() { - public void run() { - while (true) { - try { - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(settings.getMetricsLocation()); - ReaderMetricsProcessor.setCurrentInstance(mp); - mp.run(); - } catch (Throwable e) { - if (TroubleshootingConfig.getEnableDevAssert()) { - break; - } - LOG.error("Error in ReaderMetricsProcessor...restarting, ExceptionCode: {}", - StatExceptionCode.READER_RESTART_PROCESSING.toString()); - StatsCollector.instance().logException(StatExceptionCode.READER_RESTART_PROCESSING); - } - } - } - }); - readerThread.start(); - - int readerPort = getPortNumber(); - try { - Security.addProvider(new BouncyCastleProvider()); - HttpServer server = null; - if (settings.getHttpsEnabled()) { - server = createHttpsServer(readerPort); - } - else { - server = createHttpServer(readerPort); - } - server.createContext(QUERY_URL, new QueryMetricsRequestHandler()); - server.setExecutor(Executors.newCachedThreadPool()); - server.start(); - } catch (java.net.BindException ex) { - LOG.error("Port {} is already in use...exiting", readerPort); - Runtime.getRuntime().halt(1); - } catch (Exception ex) { - LOG.error("Exception in starting Reader Process: " + ex.toString()); - Runtime.getRuntime().halt(1); - } - } - - private static HttpServer createHttpsServer(int readerPort) throws Exception { - HttpsServer server = null; - String bindHost = getBindHost(); - if (bindHost != null && !bindHost.trim().isEmpty()) { - LOG.info("Binding to Interface: {}", bindHost); - server = HttpsServer.create(new InetSocketAddress(InetAddress.getByName(bindHost.trim()), readerPort), - INCOMING_QUEUE_LENGTH); - } else { - LOG.info("Value Not Configured for: {} Using default value: binding to all interfaces", WEBSERVICE_BIND_HOST_NAME); - server = HttpsServer.create(new InetSocketAddress(readerPort), INCOMING_QUEUE_LENGTH); - } - - TrustManager[] trustAllCerts = new TrustManager[] { - new X509TrustManager() { - - public X509Certificate[] getAcceptedIssuers() { - return null; - } - public void checkClientTrusted(X509Certificate[] certs, String authType) { - - } - public void checkServerTrusted(X509Certificate[] certs, String authType) { - - } - } - }; - - HostnameVerifier allHostsValid = new HostnameVerifier() { - public boolean verify(String hostname, SSLSession session) { - return true; - } - }; - - // Install the all-trusting trust manager - SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); - - KeyStore ks = CertificateUtils.createKeyStore(); - KeyManagerFactory kmf = KeyManagerFactory.getInstance("NewSunX509"); - kmf.init(ks, CertificateUtils.IN_MEMORY_PWD.toCharArray()); - sslContext.init(kmf.getKeyManagers(), trustAllCerts, null); - - HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory()); - HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); - server.setHttpsConfigurator(new HttpsConfigurator(sslContext)); - return server; - } - - private static HttpServer createHttpServer(int readerPort) throws Exception { - HttpServer server = null; - String bindHost = getBindHost(); - if (bindHost != null && !bindHost.trim().isEmpty()) { - LOG.info("Binding to Interface: {}", bindHost); - server = HttpServer.create(new InetSocketAddress(InetAddress.getByName(bindHost.trim()), readerPort), - INCOMING_QUEUE_LENGTH); - } else { - LOG.info("Value Not Configured for: {} Using default value: binding to all interfaces", WEBSERVICE_BIND_HOST_NAME); - server = HttpServer.create(new InetSocketAddress(readerPort), INCOMING_QUEUE_LENGTH); - } - - return server; - } - - private static int getPortNumber() { - String readerPortValue; - try { - readerPortValue = PluginSettings.instance().getSettingValue(WEBSERVICE_PORT_CONF_NAME); - - if (readerPortValue == null) { - LOG.info("{} not configured; using default value: {}", WEBSERVICE_PORT_CONF_NAME, WEBSERVICE_DEFAULT_PORT); - return WEBSERVICE_DEFAULT_PORT; - } - - return Integer.parseInt(readerPortValue); - } catch (Exception ex) { - LOG.error("Invalid Configuration: {} Using default value: {} AND Error: {}", - WEBSERVICE_PORT_CONF_NAME, WEBSERVICE_DEFAULT_PORT, ex.toString()); - return WEBSERVICE_DEFAULT_PORT; - } - } - - private static String getBindHost() { - return PluginSettings.instance().getSettingValue(WEBSERVICE_BIND_HOST_NAME); - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java index aa106117..eb3ad698 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java @@ -15,8 +15,6 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer; -import static java.util.Collections.singletonList; - import java.io.File; import java.security.AccessController; import java.security.PrivilegedAction; @@ -27,20 +25,6 @@ import java.util.Map; import java.util.function.Supplier; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.config.PerformanceAnalyzerConfigAction; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.action.PerformanceAnalyzerActionFilter; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.CircuitBreakerCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.HeapMetricsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceEventMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NetworkE2ECollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NetworkInterfaceCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NodeDetailsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.whoami.TransportWhoAmIAction; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.whoami.WhoAmIAction; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.listener.PerformanceAnalyzerSearchListener; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.transport.PerformanceAnalyzerTransportInterceptor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -76,22 +60,43 @@ import org.elasticsearch.transport.TransportInterceptor; import org.elasticsearch.watcher.ResourceWatcherService; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.action.PerformanceAnalyzerActionFilter; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.CircuitBreakerCollector; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.DisksCollector; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.HeapMetricsCollector; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceEventMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MetricsPurgeActivity; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NodeStatsMetricsCollector; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NetworkInterfaceCollector; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NodeDetailsCollector; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.OSMetricsCollector; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ScheduledMetricCollectorsExecutor; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ThreadPoolMetricsCollector; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.config.PerformanceAnalyzerConfigAction; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.whoami.TransportWhoAmIAction; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.whoami.WhoAmIAction; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.listener.PerformanceAnalyzerSearchListener; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.EventLog; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.transport.PerformanceAnalyzerTransportInterceptor; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.EventLogFileHandler; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.Utils; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.writer.EventLogQueueProcessor; + +import static java.util.Collections.singletonList; -public class PerformanceAnalyzerPlugin extends Plugin implements ActionPlugin, NetworkPlugin, SearchPlugin { +public final class PerformanceAnalyzerPlugin extends Plugin implements ActionPlugin, NetworkPlugin, SearchPlugin { private static final Logger LOG = LogManager.getLogger(PerformanceAnalyzerPlugin.class); public static final String PLUGIN_NAME = "opendistro_performance_analyzer"; + public static final int QUEUE_PURGE_INTERVAL_MS = 1000; private static SecurityManager sm = null; static { SecurityManager sm = System.getSecurityManager(); - + Utils.configureMetrics(); if(sm != null) { // unprivileged code such as scripts do not have SpecialPermission sm.checkPermission(new SpecialPermission()); @@ -131,24 +136,30 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa ESResources.INSTANCE.setConfigPath(configPath); ESResources.INSTANCE.setPluginFileLocation(new Environment(settings, configPath). pluginsFile().toAbsolutePath().toString() + File.separator + PLUGIN_NAME + File.separator); - //Initialize plugin settings. Accessing plugin settings before this + //initialize plugin settings. Accessing plugin settings before this //point will break, as the plugin location will not be initialized. PluginSettings.instance(); scheduledMetricCollectorsExecutor = new ScheduledMetricCollectorsExecutor(); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new ThreadPoolMetricsCollector()); - scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new NodeStatsMetricsCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new CircuitBreakerCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new OSMetricsCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new HeapMetricsCollector()); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MetricsPurgeActivity()); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new NodeDetailsCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MasterServiceMetrics()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MasterServiceEventMetrics()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new DisksCollector()); - scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new NetworkE2ECollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new NetworkInterfaceCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(StatsCollector.instance()); scheduledMetricCollectorsExecutor.start(); + + EventLog eventLog = new EventLog(); + EventLogFileHandler eventLogFileHandler = new EventLogFileHandler(eventLog, PluginSettings.instance().getMetricsLocation()); + new EventLogQueueProcessor(eventLogFileHandler, + MetricsConfiguration.SAMPLING_INTERVAL, + QUEUE_PURGE_INTERVAL_MS).scheduleExecutor(); } // - http level: bulk, search @@ -186,7 +197,8 @@ public List getRestHandlers(final Settings s final SettingsFilter settingsFilter, final IndexNameExpressionResolver indexNameExpressionResolver, final Supplier nodesInCluster) { - PerformanceAnalyzerConfigAction performanceanalyzerConfigAction = new PerformanceAnalyzerConfigAction(settings, restController); + PerformanceAnalyzerConfigAction performanceanalyzerConfigAction = new PerformanceAnalyzerConfigAction(settings, + restController, scheduledMetricCollectorsExecutor); PerformanceAnalyzerConfigAction.setInstance(performanceanalyzerConfigAction); return singletonList(performanceanalyzerConfigAction); } @@ -205,10 +217,10 @@ public Collection createComponents(Client client, ClusterService cluster @Override public Map> getTransports(Settings settings, ThreadPool threadPool, - PageCacheRecycler pageCacheRecycler, - CircuitBreakerService circuitBreakerService, - NamedWriteableRegistry namedWriteableRegistry, - NetworkService networkService) { + PageCacheRecycler pageCacheRecycler, + CircuitBreakerService circuitBreakerService, + NamedWriteableRegistry namedWriteableRegistry, + NetworkService networkService) { ESResources.INSTANCE.setSettings(settings); ESResources.INSTANCE.setCircuitBreakerService(circuitBreakerService); return Collections.emptyMap(); diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CachedStats.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CachedStats.java deleted file mode 100644 index accd1b05..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CachedStats.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardStatsValue; - -class CachedStats { - private static final Set CACHABLE_VALUES = new HashSet<>(Arrays.asList( - ShardStatsValue.INDEXING_THROTTLE_TIME.toString(), - ShardStatsValue.CACHE_QUERY_HIT.toString(), - ShardStatsValue.CACHE_QUERY_MISS.toString(), - ShardStatsValue.CACHE_FIELDDATA_EVICTION.toString(), - ShardStatsValue.CACHE_REQUEST_HIT.toString(), - ShardStatsValue.CACHE_REQUEST_MISS.toString(), - ShardStatsValue.CACHE_REQUEST_EVICTION.toString(), - ShardStatsValue.REFRESH_EVENT.toString(), - ShardStatsValue.REFRESH_TIME.toString(), - ShardStatsValue.FLUSH_EVENT.toString(), - ShardStatsValue.FLUSH_TIME.toString(), - ShardStatsValue.MERGE_EVENT.toString(), - ShardStatsValue.MERGE_TIME.toString() - )); - private Map cachedValues = new HashMap<>(); - - long getValue(String statsName) { - return cachedValues.getOrDefault(statsName, 0L); - } - - void putValue(String statsName, long value) { - cachedValues.put(statsName, value); - } - - static Set getCachableValues() { - return CACHABLE_VALUES; - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/DiskMetrics.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/DiskMetrics.java deleted file mode 100644 index 231da7d6..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/DiskMetrics.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.DiskDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.DiskValue; -import com.fasterxml.jackson.annotation.JsonProperty; - -public class DiskMetrics extends MetricStatus { - public String name; - - public double utilization; // fraction, 0-1 - - public double await; // ms - - public double serviceRate; // MBps - - public DiskMetrics(String name, double utilization, double await, - double serviceRate) { - super(); - this.name = name; - this.utilization = utilization; - this.await = await; - this.serviceRate = serviceRate; - } - - public DiskMetrics() { - super(); - } - - @JsonProperty(DiskDimension.Constants.NAME_VALUE) - public String getName() { - return name; - } - - @JsonProperty(DiskValue.Constants.UTIL_VALUE) - public double getUtilization() { - return utilization; - } - - @JsonProperty(DiskValue.Constants.WAIT_VALUE) - public double getAwait() { - return await; - } - - @JsonProperty(DiskValue.Constants.SRATE_VALUE) - public double getServiceRate() { - return serviceRate; - } -} \ No newline at end of file diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/DisksCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/DisksCollector.java deleted file mode 100644 index eadbdfa6..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/DisksCollector.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import java.util.HashMap; -import java.util.Map; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.OSMetricsGeneratorFactory; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.DiskMetricsGenerator; - -public class DisksCollector extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor { - - private static final int sTimeInterval = MetricsConfiguration.CONFIG_MAP - .get(DisksCollector.class).samplingInterval; - private StringBuilder value = new StringBuilder(); - - public DisksCollector() { - super(sTimeInterval, "DisksCollector"); - } - - @Override - public String getMetricsPath(long startTime, String... keysPath) { - // throw exception if keys.length is not equal to 0 - if (keysPath.length != 0) { - throw new RuntimeException("keys length should be 0"); - } - - return PerformanceAnalyzerMetrics.generatePath(startTime, PerformanceAnalyzerMetrics.sDisksPath); - } - - @Override - public void collectMetrics(long startTime) { - DiskMetricsGenerator diskMetricsGenerator = OSMetricsGeneratorFactory.getInstance().getDiskMetricsGenerator(); - diskMetricsGenerator.addSample(); - - saveMetricValues(getMetrics(diskMetricsGenerator), startTime); - } - - private Map getMetricsMap(DiskMetricsGenerator diskMetricsGenerator) { - - Map map = new HashMap<>(); - - for (String disk : diskMetricsGenerator.getAllDisks()) { - DiskMetrics diskMetrics = new DiskMetrics(); - diskMetrics.name = disk; - diskMetrics.await = diskMetricsGenerator.getAwait(disk); - diskMetrics.serviceRate = diskMetricsGenerator.getServiceRate(disk); - diskMetrics.utilization = diskMetricsGenerator.getDiskUtilization(disk); - - map.put(disk, diskMetrics); - } - - return map; - } - - private String getMetrics(DiskMetricsGenerator diskMetricsGenerator) { - - Map map = getMetricsMap(diskMetricsGenerator); - value.setLength(0); - value.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds()) - .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); - for (Map.Entry entry : map.entrySet()) { - value.append(entry.getValue().serialize()) - .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); - } - return value.toString(); - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/HeapMetricsCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/HeapMetricsCollector.java deleted file mode 100644 index ced57d6f..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/HeapMetricsCollector.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import java.lang.management.MemoryUsage; -import java.util.Map; -import java.util.function.Supplier; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm.GCMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm.HeapMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.GCType; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HeapDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HeapValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.fasterxml.jackson.annotation.JsonProperty; - -import com.fasterxml.jackson.annotation.JsonProperty; - - -public class HeapMetricsCollector extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor { - private static final Logger LOG = LogManager.getLogger(HeapMetricsCollector.class); - public static final int SAMPLING_TIME_INTERVAL = MetricsConfiguration.CONFIG_MAP.get(HeapMetricsCollector.class).samplingInterval; - private static final int KEYS_PATH_LENGTH = 0; - private StringBuilder value; - - - public HeapMetricsCollector() { - super(SAMPLING_TIME_INTERVAL, "HeapMetrics"); - value = new StringBuilder(); - } - - @Override - public void collectMetrics(long startTime) { - GCMetrics.runGCMetrics(); - - value.setLength(0); - value.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds()) - .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); - value.append(new HeapStatus(GCType.TOT_YOUNG_GC.toString(), - GCMetrics.getTotYoungGCCollectionCount(), - GCMetrics.getTotYoungGCCollectionTime()).serialize()).append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); - - value.append(new HeapStatus(GCType.TOT_FULL_GC.toString(), - GCMetrics.getTotFullGCCollectionCount(), - GCMetrics.getTotFullGCCollectionTime()).serialize()).append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); - - for (Map.Entry> entry : HeapMetrics - .getMemoryUsageSuppliers().entrySet()) { - MemoryUsage memoryUsage = entry.getValue().get(); - - value.append( - new HeapStatus(entry.getKey(), - memoryUsage.getCommitted(), - memoryUsage.getInit(), - memoryUsage.getMax(), - memoryUsage.getUsed()).serialize()).append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); - } - - saveMetricValues(value.toString(), startTime); - } - - @Override - public String getMetricsPath(long startTime, String... keysPath) { - // throw exception if keys.length is not equal to 0 - if (keysPath.length != KEYS_PATH_LENGTH) { - throw new RuntimeException("keys length should be " + KEYS_PATH_LENGTH); - } - - return PerformanceAnalyzerMetrics.generatePath(startTime, PerformanceAnalyzerMetrics.sHeapPath); - } - - public static class HeapStatus extends MetricStatus { - // GC type like survivor - private final String type; - - // -2 means this metric is undefined for a memory pool. For example, - // The memory pool Eden has no collectionCount metric. - private static final long UNDEFINED = -2; - - // the total number of collections that have occurred - private long collectionCount = UNDEFINED; - - // the approximate accumulated collection elapsed time in milliseconds - private long collectionTime = UNDEFINED; - - // the amount of memory in bytes that is committed for the Java virtual machine to use - private long committed = UNDEFINED; - - // the amount of memory in bytes that the Java virtual machine initially requests from the operating system for memory management - private long init = UNDEFINED; - - // the maximum amount of memory in bytes that can be used for memory management - private long max = UNDEFINED; - - // the amount of used memory in bytes - private long used = UNDEFINED; - - public HeapStatus(String type, - long collectionCount, - long collectionTime) { - - - this.type = type; - this.collectionCount = collectionCount; - this.collectionTime = collectionTime; - } - - public HeapStatus(String type, - long committed, - long init, - long max, - long used) { - - this.type = type; - this.committed = committed; - this.init = init; - this.max = max; - this.used = used; - - } - - @JsonProperty(HeapDimension.Constants.TYPE_VALUE) - public String getType() { - return type; - } - - @JsonProperty(HeapValue.Constants.COLLECTION_COUNT_VALUE) - public long getCollectionCount() { - return collectionCount; - } - - @JsonProperty(HeapValue.Constants.COLLECTION_TIME_VALUE) - public long getCollectionTime() { - return collectionTime; - } - - @JsonProperty(HeapValue.Constants.COMMITTED_VALUE) - public long getCommitted() { - return committed; - } - - @JsonProperty(HeapValue.Constants.INIT_VALUE) - public long getInit() { - return init; - } - - @JsonProperty(HeapValue.Constants.MAX_VALUE) - public long getMax() { - return max; - } - - @JsonProperty(HeapValue.Constants.USED_VALUE) - public long getUsed() { - return used; - } - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceEventMetrics.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceEventMetrics.java index 24bca574..6bf68c75 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceEventMetrics.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceEventMetrics.java @@ -18,12 +18,11 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Queue; -import java.util.HashSet; import java.util.concurrent.ThreadPoolExecutor; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.cluster.service.MasterService; @@ -35,15 +34,15 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterMetricValues; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.ThreadIDUtil; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; @SuppressWarnings("unchecked") public class MasterServiceEventMetrics extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor { public static final int SAMPLING_TIME_INTERVAL = MetricsConfiguration.CONFIG_MAP.get( MasterServiceEventMetrics.class).samplingInterval; private static final Logger LOG = LogManager.getLogger(MasterServiceEventMetrics.class); + private static final String MASTER_NODE_NOT_UP_METRIC = "MasterNodeNotUp"; private long lastTaskInsertionOrder; private static final int KEYS_PATH_LENGTH = 3; private StringBuilder value; @@ -193,8 +192,12 @@ Queue getMasterServiceCurrentQueue() throws Exception { (PrioritizedEsThreadPoolExecutor) getMasterServiceTPExecutorField().get(masterService); } - masterServiceCurrentQueue = - (Queue) getPrioritizedTPExecutorCurrentField().get(prioritizedEsThreadPoolExecutor); + if (prioritizedEsThreadPoolExecutor != null) { + masterServiceCurrentQueue = + (Queue) getPrioritizedTPExecutorCurrentField().get(prioritizedEsThreadPoolExecutor); + } else { + StatsCollector.instance().logMetric(MASTER_NODE_NOT_UP_METRIC); + } } } } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetrics.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetrics.java index 797e1ed5..80a8123e 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetrics.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetrics.java @@ -84,4 +84,3 @@ public int getPendingTasksCount() { } } } - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MetricStatus.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MetricStatus.java deleted file mode 100644 index 7e33f198..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MetricStatus.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.JsonConverter; - -public class MetricStatus { - - /** - * converts any object to a JSON string and return that string - * @return A string containing a JSON representation of the object - */ - public String serialize() { - return JsonConverter.writeValueAsString(this); - } - -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MetricsPurgeActivity.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MetricsPurgeActivity.java deleted file mode 100644 index 05da2d28..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MetricsPurgeActivity.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import java.io.File; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; - -public class MetricsPurgeActivity extends PerformanceAnalyzerMetricsCollector { - public MetricsPurgeActivity() { - super(MetricsConfiguration.CONFIG_MAP.get(MetricsPurgeActivity.class).samplingInterval, - "MetricsPurgeActivity"); - } - - private static int purgeInterval = MetricsConfiguration.CONFIG_MAP.get(MetricsPurgeActivity.class).deletionInterval; - - @Override - public void collectMetrics(long startTime) { - File root = new File(PluginSettings.instance().getMetricsLocation()); - - String[] children = root.list(); - if (children == null) { - return; - } - for (int i = 0; i < children.length; i++) { - if (Long.parseLong(children[i]) < PerformanceAnalyzerMetrics.getTimeInterval(startTime - purgeInterval)) { - PerformanceAnalyzerMetrics.removeMetrics(new File(root, children[i])); - } - } - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NetInterfaceSummary.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NetInterfaceSummary.java deleted file mode 100644 index 9c4a5b3d..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NetInterfaceSummary.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPValue; -import com.fasterxml.jackson.annotation.JsonProperty; - -// all metrics are per-time-unit -public class NetInterfaceSummary extends MetricStatus { - - public enum Direction { - in, out; - } - - private Direction direction; - private double packetRate4; - private double dropRate4; - private double packetRate6; - private double dropRate6; - private double bps; - - public NetInterfaceSummary(Direction direction, - double packetRate4, - double dropRate4, - double packetRate6, - double dropRate6, - double bps) { - this.direction = direction; - this.packetRate4 = packetRate4; - this.dropRate4 = dropRate4; - this.packetRate6 = packetRate6; - this.dropRate6 = dropRate6; - this.bps = bps; - } - - @JsonProperty(IPDimension.Constants.DIRECTION_VALUE) - public Direction getDirection() { - return direction; - } - - @JsonProperty(IPValue.Constants.PACKET_RATE4_VALUE) - public double getPacketRate4() { - return packetRate4; - } - - @JsonProperty(IPValue.Constants.DROP_RATE4_VALUE) - public double getDropRate4() { - return dropRate4; - } - - @JsonProperty(IPValue.Constants.PACKET_RATE6_VALUE) - public double getPacketRate6() { - return packetRate6; - } - - @JsonProperty(IPValue.Constants.DROP_RATE6_VALUE) - public double getDropRate6() { - return dropRate6; - } - - @JsonProperty(IPValue.Constants.THROUGHPUT_VALUE) - public double getBps() { - return bps; - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NetworkE2ECollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NetworkE2ECollector.java deleted file mode 100644 index dff6e89f..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NetworkE2ECollector.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import java.util.HashMap; -import java.util.Map; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.OSMetricsGeneratorFactory; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.TCPMetricsGenerator; - -public class NetworkE2ECollector extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor { - private static final int sTimeInterval = MetricsConfiguration.CONFIG_MAP.get(NetworkE2ECollector.class).samplingInterval; - - - public NetworkE2ECollector() { - super(sTimeInterval, "NetworkE2ECollector"); - } - - @Override - public void collectMetrics(long startTime) { - TCPMetricsGenerator tcpMetricsGenerator = OSMetricsGeneratorFactory.getInstance().getTCPMetricsGenerator(); - tcpMetricsGenerator.addSample(); - - String value = PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds() - + PerformanceAnalyzerMetrics.sMetricNewLineDelimitor - + getMetrics(tcpMetricsGenerator); - - saveMetricValues(value, startTime); - } - - @Override - public String getMetricsPath(long startTime, String... keysPath) { - // throw exception if keys.length is not equal to 0 - if (keysPath.length != 0) { - throw new RuntimeException("keys length should be 0"); - } - - return PerformanceAnalyzerMetrics.generatePath(startTime, PerformanceAnalyzerMetrics.sTCPPath); - } - - private Map getMetricsMap(TCPMetricsGenerator tcpMetricsGenerator) { - Map map = new HashMap<>(); - - for (String dest : tcpMetricsGenerator.getAllDestionationIps()) { - TCPStatus tcpStatus = new TCPStatus( - dest, - tcpMetricsGenerator.getNumberOfFlows(dest), - tcpMetricsGenerator.getTransmitQueueSize(dest), - tcpMetricsGenerator.getReceiveQueueSize(dest), - tcpMetricsGenerator.getCurrentLost(dest), - tcpMetricsGenerator.getSendCongestionWindow(dest), - tcpMetricsGenerator.getSlowStartThreshold(dest) - ); - - map.put(dest, tcpStatus); - } - - return map; - } - - private String getMetrics(TCPMetricsGenerator tcpMetricsGenerator) { - - Map map = getMetricsMap(tcpMetricsGenerator); - StringBuilder value = new StringBuilder(); - value.setLength(0); - for (TCPStatus tcpStatus : map.values()) { - - value.append(tcpStatus.serialize()) - .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); - } - - return value.toString(); - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NetworkInterfaceCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NetworkInterfaceCollector.java deleted file mode 100644 index 899fa459..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NetworkInterfaceCollector.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.OSMetricsGeneratorFactory; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.IPMetricsGenerator; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; - - -public class NetworkInterfaceCollector extends PerformanceAnalyzerMetricsCollector -implements MetricsProcessor { - private static final int sTimeInterval = MetricsConfiguration.CONFIG_MAP.get(NetworkInterfaceCollector.class).samplingInterval; - private static final Logger LOG = LogManager.getLogger(NetworkInterfaceCollector.class); - private StringBuilder ret = new StringBuilder(); - - public NetworkInterfaceCollector() { - super(sTimeInterval, "NetworkInterfaceCollector"); - } - - @Override - public void collectMetrics(long startTime) { - - IPMetricsGenerator IPMetricsGenerator = OSMetricsGeneratorFactory.getInstance().getIPMetricsGenerator(); - IPMetricsGenerator.addSample(); - - saveMetricValues(getMetrics(IPMetricsGenerator) + PerformanceAnalyzerMetrics.sMetricNewLineDelimitor, startTime); - } - - @Override - public String getMetricsPath(long startTime, String... keysPath) { - // throw exception if keys.length is not equal to 0 - if (keysPath.length != 0) { - throw new RuntimeException("keys length should be 0"); - } - - return PerformanceAnalyzerMetrics.generatePath(startTime, PerformanceAnalyzerMetrics.sIPPath); - } - - - private String getMetrics(IPMetricsGenerator IPMetricsGenerator) { - - ret.setLength(0); - ret.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds()) - .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); - - try { - NetInterfaceSummary inNetwork = new NetInterfaceSummary( - NetInterfaceSummary.Direction.in, - IPMetricsGenerator.getInPacketRate4(), - IPMetricsGenerator.getInDropRate4(), - IPMetricsGenerator.getInPacketRate6(), - IPMetricsGenerator.getInDropRate6(), - IPMetricsGenerator.getInBps()); - - NetInterfaceSummary outNetwork = new NetInterfaceSummary( - NetInterfaceSummary.Direction.out, - IPMetricsGenerator.getOutPacketRate4(), - IPMetricsGenerator.getOutDropRate4(), - IPMetricsGenerator.getOutPacketRate6(), - IPMetricsGenerator.getOutDropRate6(), - IPMetricsGenerator.getOutBps()); - - ret.append(inNetwork.serialize()).append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); - ret.append(outNetwork.serialize()).append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); - }catch (Exception e) { - LOG.debug("Exception in NetworkInterfaceCollector: {} with ExceptionCode: {}", - () -> e.toString(), () -> StatExceptionCode.NETWORK_COLLECTION_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.NETWORK_COLLECTION_ERROR); - } - - return ret.toString(); - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/OSMetricsCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/OSMetricsCollector.java deleted file mode 100644 index ffe8c92c..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/OSMetricsCollector.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import java.util.Map; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.OSMetricsGeneratorFactory; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm.ThreadList; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.OSMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.CPUPagingActivityGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.DiskIOMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.OSMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.SchedMetricsGenerator; - -public class OSMetricsCollector extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor { - public static final int SAMPLING_TIME_INTERVAL = MetricsConfiguration.CONFIG_MAP.get(ThreadList.class).samplingInterval; - private static final int KEYS_PATH_LENGTH = 1; - private StringBuilder value; - private OSMetricsGenerator osMetricsGenerator; - - public enum MetaDataFields { - threadName - } - - public OSMetricsCollector() { - super(SAMPLING_TIME_INTERVAL, "OSMetrics"); - value = new StringBuilder(); - osMetricsGenerator = OSMetricsGeneratorFactory.getInstance(); - } - - @Override - public void collectMetrics(long startTime) { - - CPUPagingActivityGenerator threadCPUPagingActivityGenerator = osMetricsGenerator.getPagingActivityGenerator(); - threadCPUPagingActivityGenerator.addSample(); - - SchedMetricsGenerator schedMetricsGenerator = osMetricsGenerator.getSchedMetricsGenerator(); - schedMetricsGenerator.addSample(); - - Map threadStates = ThreadList.getNativeTidMap(); - - DiskIOMetricsGenerator diskIOMetricsGenerator = osMetricsGenerator.getDiskIOMetricsGenerator(); - diskIOMetricsGenerator.addSample(); - - for (String threadId : osMetricsGenerator.getAllThreadIds()) { - value.setLength(0); - value.append(PerformanceAnalyzerMetrics.getCurrentTimeMetric()) - .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.CPU_UTILIZATION) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(threadCPUPagingActivityGenerator - .getCPUUtilization(threadId)); - - if (threadCPUPagingActivityGenerator.hasPagingActivity(threadId)) { - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.PAGING_MAJ_FLT_RATE) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(threadCPUPagingActivityGenerator - .getMajorFault(threadId)); - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.PAGING_MIN_FLT_RATE) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(threadCPUPagingActivityGenerator - .getMinorFault(threadId)); - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.PAGING_RSS) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(threadCPUPagingActivityGenerator - .getResidentSetSize(threadId)); - } - - if (schedMetricsGenerator.hasSchedMetrics(threadId)) { - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.SCHED_RUNTIME) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(schedMetricsGenerator.getAvgRuntime(threadId)); - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.SCHED_WAITTIME) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(schedMetricsGenerator.getAvgWaittime(threadId)); - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.SCHED_CTX_RATE) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(schedMetricsGenerator - .getContextSwitchRate(threadId)); - } - - ThreadList.ThreadState threadState = threadStates - .get(Long.valueOf(threadId)); - if (threadState != null) { - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.HEAP_ALLOC_RATE) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(threadState.heapAllocRate); - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(MetaDataFields.threadName.toString()) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(threadState.threadName); - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.THREAD_BLOCKED_TIME) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(threadState.avgBlockedTime); - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.THREAD_BLOCKED_EVENT) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(threadState.blockedCount); - } - - if (diskIOMetricsGenerator.hasDiskIOMetrics(threadId)) { - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.IO_READ_THROUGHPUT) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(diskIOMetricsGenerator - .getAvgReadThroughputBps(threadId)); - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.IO_WRITE_THROUGHPUT) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(diskIOMetricsGenerator - .getAvgWriteThroughputBps(threadId)); - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.IO_TOT_THROUGHPUT) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(diskIOMetricsGenerator - .getAvgTotalThroughputBps(threadId)); - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.IO_READ_SYSCALL_RATE) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(diskIOMetricsGenerator - .getAvgReadSyscallRate(threadId)); - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.IO_WRITE_SYSCALL_RATE) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(diskIOMetricsGenerator - .getAvgWriteSyscallRate(threadId)); - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) - .append(OSMetrics.IO_TOTAL_SYSCALL_RATE) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(diskIOMetricsGenerator - .getAvgTotalSyscallRate(threadId)); - } - - saveMetricValues(value.toString(), startTime, threadId); - } - } - - @Override - public String getMetricsPath(long startTime, String... keysPath) { - // throw exception if keys.length is not equal to 1...which is thread ID - if (keysPath.length != KEYS_PATH_LENGTH) { - throw new RuntimeException("keys length should be " + KEYS_PATH_LENGTH); - } - return PerformanceAnalyzerMetrics.generatePath(startTime, PerformanceAnalyzerMetrics.sThreadsPath, - keysPath[0], PerformanceAnalyzerMetrics.sOSPath); - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/PerformanceAnalyzerMetricsCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/PerformanceAnalyzerMetricsCollector.java deleted file mode 100644 index 0e66832a..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/PerformanceAnalyzerMetricsCollector.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerPlugin; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import java.util.concurrent.atomic.AtomicBoolean; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; - -public abstract class PerformanceAnalyzerMetricsCollector implements Runnable { - private static final Logger LOG = LogManager.getLogger(PerformanceAnalyzerMetricsCollector.class); - private int timeInterval; - private long startTime; - private String collectorName; - - protected PerformanceAnalyzerMetricsCollector(int timeInterval, String collectorName) { - this.timeInterval = timeInterval; - this.collectorName = collectorName; - } - - private AtomicBoolean bInProgress = new AtomicBoolean(false); - - public int getTimeInterval() { - return timeInterval; - } - - public boolean inProgress() { - return bInProgress.get(); - } - - public String getCollectorName() { - return collectorName; - } - - abstract void collectMetrics(long startTime); - - public void setStartTime(long startTime) { - this.startTime = startTime; - bInProgress.set(true); - } - - public void run() { - try { - PerformanceAnalyzerPlugin.invokePrivileged(() -> collectMetrics(startTime)); - } catch (Exception ex) { - //- should not be any...but in case, absorbing here - //- logging...we shouldn't be doing as it will slow down; as well as fill up the log. Need to find a way to catch these - LOG.debug("Error In Collect Metrics: {} with ExceptionCode: {}", - () -> ex.toString(), () -> StatExceptionCode.OTHER_COLLECTION_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.OTHER_COLLECTION_ERROR); - } finally { - bInProgress.set(false); - } - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ScheduledMetricCollectorsExecutor.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ScheduledMetricCollectorsExecutor.java deleted file mode 100644 index 783ace0e..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ScheduledMetricCollectorsExecutor.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import java.util.Map; -import java.util.HashMap; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.config.PerformanceAnalyzerConfigAction; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - - -public class ScheduledMetricCollectorsExecutor extends Thread { - private static final Logger LOG = LogManager.getLogger(ScheduledMetricCollectorsExecutor.class); - private final int collectorThreadCount; - private static final int DEFAULT_COLLECTOR_THREAD_COUNT = 5; - private static final int COLLECTOR_THREAD_KEEPALIVE_SECS = 1000; - private final boolean checkFeatureDisabledFlag; - - private int minTimeIntervalToSleep = Integer.MAX_VALUE; - private Map metricsCollectors; - private ThreadPoolExecutor metricsCollectorsTP; - - public ScheduledMetricCollectorsExecutor(int collectorThreadCount, boolean checkFeatureDisabledFlag) { - metricsCollectors = new HashMap<>(); - metricsCollectorsTP = null; - this.collectorThreadCount = collectorThreadCount; - this.checkFeatureDisabledFlag = checkFeatureDisabledFlag; - } - - public ScheduledMetricCollectorsExecutor() { - this(DEFAULT_COLLECTOR_THREAD_COUNT, true); - } - - public void addScheduledMetricCollector(PerformanceAnalyzerMetricsCollector task) { - metricsCollectors.put(task, System.currentTimeMillis() + task.getTimeInterval()); - if (task.getTimeInterval() < minTimeIntervalToSleep) { - minTimeIntervalToSleep = task.getTimeInterval(); - } - } - - public void run() { - if (metricsCollectorsTP == null) { - metricsCollectorsTP = new ThreadPoolExecutor(collectorThreadCount, - collectorThreadCount, - COLLECTOR_THREAD_KEEPALIVE_SECS, - TimeUnit.SECONDS, - new ArrayBlockingQueue<>(metricsCollectors.size())); - } - - long prevStartTimestamp = System.currentTimeMillis(); - - while (true) { - try { - long millisToSleep = minTimeIntervalToSleep - System.currentTimeMillis() + prevStartTimestamp; - if (millisToSleep > 0) { - Thread.sleep(millisToSleep); - } - } catch (Exception ex) { - LOG.error("Exception in Thread Sleep", ex); - } - - prevStartTimestamp = System.currentTimeMillis(); - - if (!checkFeatureDisabledFlag || - PerformanceAnalyzerConfigAction.getInstance() != null && PerformanceAnalyzerConfigAction.getInstance().isFeatureEnabled()) { - long currentTime = System.currentTimeMillis(); - - for (Map.Entry entry : metricsCollectors.entrySet()) { - if (entry.getValue() <= currentTime) { - PerformanceAnalyzerMetricsCollector collector = entry.getKey(); - metricsCollectors.put(collector, entry.getValue() + collector.getTimeInterval()); - if (!collector.inProgress()) { - collector.setStartTime(currentTime); - metricsCollectorsTP.execute(collector); - } else { - LOG.info("Collector {} is still in progress, so skipping this Interval", collector.getCollectorName()); - } - } - } - } - } - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/StatExceptionCode.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/StatExceptionCode.java deleted file mode 100644 index 2429792c..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/StatExceptionCode.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -public enum StatExceptionCode { - TOTAL_ERROR("TotalError"), - METRICS_WRITE_ERROR("MetricsWriteError"), - METRICS_REMOVE_ERROR("MetricsRemoveError"), - JVM_ATTACH_ERROR("JvmAttachErrror"), - MASTER_METRICS_ERROR("MasterMetricsError"), - DISK_METRICS_ERROR("DiskMetricsError"), - THREAD_IO_ERROR("ThreadIOError"), - SCHEMA_PARSER_ERROR("SchemaParserError"), - JSON_PARSER_ERROR("JsonParserError"), - NETWORK_COLLECTION_ERROR("NetworkCollectionError"), - NODESTATS_COLLECTION_ERROR("NodeStatsCollectionError"), - OTHER_COLLECTION_ERROR("OtherCollectionError"), - REQUEST_ERROR("RequestError"), - REQUEST_REMOTE_ERROR("RequestRemoteError"), - READER_PARSER_ERROR("ReaderParserError"), - READER_RESTART_PROCESSING("ReaderRestartProcessing"), - OTHER("Other"); - - private final String value; - - StatExceptionCode(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/StatsCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/StatsCollector.java deleted file mode 100644 index f026c332..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/StatsCollector.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.concurrent.ConcurrentHashMap; -import java.util.Locale; -import java.util.Map; -import java.util.List; -import java.util.Vector; -import java.util.Properties; - -import java.io.InputStream; -import java.io.FileInputStream; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import java.util.concurrent.atomic.AtomicInteger; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.google.common.annotations.VisibleForTesting; - -public class StatsCollector extends PerformanceAnalyzerMetricsCollector { - private static final String LOG_ENTRY_INIT = "------------------------------------------------------------------------"; - private static final String LOG_ENTRY_END = "EOE"; - private static final String LOG_LINE_BREAK = "\n"; - private static final double MILLISECONDS_TO_SECONDS_DIVISOR = 1000D; - - private static final Logger STATS_LOGGER = LogManager.getLogger("stats_log"); - private static final Logger GENERAL_LOG = LogManager.getLogger(StatsCollector.class); - private static StatsCollector statsCollector = null; - public static String STATS_TYPE = "plugin-stats-metadata"; - - private final Map metadata; - private Map counters = new ConcurrentHashMap<>(); - private Date objectCreationTime = new Date(); - - private List defaultExceptionCodes = new Vector<>(); - - public static StatsCollector instance() { - if(statsCollector == null) { - synchronized(StatsCollector.class) { - if(statsCollector == null) { - statsCollector = new StatsCollector(loadMetadata(PluginSettings.instance().getSettingValue(STATS_TYPE, STATS_TYPE))); - } - } - } - - return statsCollector; - } - - @VisibleForTesting - Map getCounters() { - return counters; - } - public void logException() { - logException(StatExceptionCode.OTHER); - } - - public void logException(StatExceptionCode statExceptionCode) { - incCounter(statExceptionCode.toString()); - } - - public void logStatsRecord(Map counters, Map statsdata, - Map latencies, long startTimeMillis, long endTimeMillis) { - writeStats(metadata, counters, statsdata, latencies, startTimeMillis, endTimeMillis); - } - - private static Map loadMetadata(String fileLocation) { - Map retVal = new ConcurrentHashMap<>(); - - if(fileLocation != null) { - Properties props = new Properties(); - - try (InputStream input = new FileInputStream( - ESResources.INSTANCE.getPluginFileLocation() + PluginSettings.CONFIG_FILES_PATH + fileLocation); ) { - // load properties file - props.load(input); - } catch(Exception ex) { - GENERAL_LOG.error("Error in loading metadata for fileLocation: {}", fileLocation); - } - - props.forEach((key, value) -> retVal.put((String)key, (String)value)); - } - - return retVal; - } - - private StatsCollector(Map metadata) { - super(MetricsConfiguration.CONFIG_MAP.get(StatsCollector.class).samplingInterval, - "StatsCollector"); - this.metadata = metadata; - defaultExceptionCodes.add(StatExceptionCode.TOTAL_ERROR); - } - - public void addDefaultExceptionCode(StatExceptionCode statExceptionCode) { - defaultExceptionCodes.add(statExceptionCode); - } - - @Override - public void collectMetrics(long startTime) { - Map currentCounters = counters; - counters = new ConcurrentHashMap<>(); - - //currentCounters.putIfAbsent(StatExceptionCode.TOTAL_ERROR.toString(), new AtomicInteger(0)); - - for(StatExceptionCode statExceptionCode : defaultExceptionCodes) { - currentCounters.putIfAbsent(statExceptionCode.toString(), new AtomicInteger(0)); - } - - writeStats(metadata, currentCounters, null, null, objectCreationTime.getTime(), new Date().getTime()); - objectCreationTime = new Date(); - } - - private void incCounter(String counterName) { - AtomicInteger val = counters.putIfAbsent(counterName, new AtomicInteger(1)); - if (val != null) { - val.getAndIncrement(); - } - - AtomicInteger all_val = counters.putIfAbsent(StatExceptionCode.TOTAL_ERROR.toString(), new AtomicInteger(1)); - if (all_val != null) { - all_val.getAndIncrement(); - } - } - - private static void writeStats(Map metadata, Map counters, - Map statsdata, Map latencies, - long startTimeMillis, long endTimeMillis) { - StringBuilder builder = new StringBuilder(); - builder.append(LOG_ENTRY_INIT + LOG_LINE_BREAK); - logValues(metadata, builder); - logValues(statsdata, builder); - logTimeMetrics(startTimeMillis, endTimeMillis, builder); - - Map tmpLatencies; - - if(latencies == null) { - tmpLatencies = new ConcurrentHashMap<>(); - } else { - tmpLatencies = new ConcurrentHashMap<>(latencies); - } - - tmpLatencies.put("total-time", (double)endTimeMillis-startTimeMillis); - addEntry("Timing", getLatencyMetrics(tmpLatencies), builder); - - - addEntry("Counters", getCountersString(counters), builder); - builder.append(LOG_ENTRY_END);// + LOG_LINE_BREAK); - STATS_LOGGER.info(builder.toString()); - } - - private static String getCountersString(Map counters) { - StringBuilder builder = new StringBuilder(); - if (counters == null || counters.isEmpty()) { - return ""; - } - for (Map.Entry counter : counters.entrySet()) { - builder.append(counter.getKey()).append("=").append(counter.getValue().get()).append(","); - } - builder.delete(builder.length() - 1, builder.length()); - return builder.toString(); - } - - private static void logTimeMetrics(long startTimeMillis, long endTimeMillis, StringBuilder builder) { - // Date Example: Wed, 20 Mar 2013 15:07:51 GMT - SimpleDateFormat dateFormat = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.ROOT); - addEntry("StartTime", String.format(Locale.ROOT, "%.3f", startTimeMillis / MILLISECONDS_TO_SECONDS_DIVISOR), builder); - addEntry("EndTime", dateFormat.format(new Date(endTimeMillis)), builder); - addEntry("Time", (endTimeMillis - startTimeMillis) + " msecs", builder); - } - - private static void logValues(Map values, StringBuilder sb) { - if(values == null) { - return; - } - for (Map.Entry entry : values.entrySet()) { - addEntry(entry.getKey(), entry.getValue(), sb); - } - } - - private static void addEntry(String key, Object value, StringBuilder sb) { - sb.append(key).append('=').append(value).append(LOG_LINE_BREAK); - } - - private static String getLatencyMetrics(Map values) { - StringBuilder builder = new StringBuilder(); - if (values == null || values.isEmpty()) { - return ""; - } - for (Map.Entry value : values.entrySet()) { - getTimingInfo(value.getKey(), value.getValue(), builder); - } - builder.delete(builder.length() - 1, builder.length()); - return builder.toString(); - } - - private static void getTimingInfo(String timerName, double latency, StringBuilder builder) { - getTimingInfo(timerName, latency, builder, 1); - } - - private static void getTimingInfo(String timerName, double latency, StringBuilder builder, int attempts) { - builder.append(timerName).append(":").append(latency).append("/").append(attempts).append(","); - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/TCPStatus.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/TCPStatus.java deleted file mode 100644 index b3228abb..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/TCPStatus.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.TCPDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.TCPValue; -import com.fasterxml.jackson.annotation.JsonProperty; - -public class TCPStatus extends MetricStatus { - - private String dest; - - private int numFlows; - - private double txQ; - - private double rxQ; - - private double curLost; - - private double sndCWND; - - // make this field private so that Jackson uses getter method name - private double ssThresh; - - public TCPStatus(String dest, int numFlows, double txQ, double rxQ, - double curLost, double sndCWND, double sSThresh) { - super(); - this.dest = dest; - this.numFlows = numFlows; - this.txQ = txQ; - this.rxQ = rxQ; - this.curLost = curLost; - this.sndCWND = sndCWND; - this.ssThresh = sSThresh; - } - - @JsonProperty(TCPDimension.Constants.DEST_VALUE) - public String getDest() { - return dest; - } - - @JsonProperty(TCPValue.Constants.NUM_FLOWS_VALUE) - public int getNumFlows() { - return numFlows; - } - - @JsonProperty(TCPValue.Constants.TXQ_VALUE) - public double getTxQ() { - return txQ; - } - - @JsonProperty(TCPValue.Constants.RXQ_VALUE) - public double getRxQ() { - return rxQ; - } - - @JsonProperty(TCPValue.Constants.CUR_LOST_VALUE) - public double getCurLost() { - return curLost; - } - - @JsonProperty(TCPValue.Constants.SEND_CWND_VALUE) - public double getSndCWND() { - return sndCWND; - } - - @JsonProperty(TCPValue.Constants.SSTHRESH_VALUE) - public double getSSThresh() { - return ssThresh; - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/ConfigFatalException.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/ConfigFatalException.java deleted file mode 100644 index c87a6984..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/ConfigFatalException.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.config; - -class ConfigFatalException extends Exception { - ConfigFatalException(String message) { - super(message); - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/ConfigFileException.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/ConfigFileException.java deleted file mode 100644 index 9c88cc5f..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/ConfigFileException.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.config; - -class ConfigFileException extends Exception { - ConfigFileException(Throwable cause) { - super(cause); - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PluginSettings.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PluginSettings.java deleted file mode 100644 index 07d67e9e..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PluginSettings.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.config; - -import java.io.File; -import java.util.Properties; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.logging.log4j.util.Supplier; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.ConfigStatus; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerPlugin; - -public class PluginSettings { - private static final Logger LOG = LogManager.getLogger(PluginSettings.class); - - private static PluginSettings instance; - public static final String CONFIG_FILES_PATH = "pa_config/"; - private static final String DEFAULT_CONFIG_FILE_PATH = "pa_config/performance-analyzer.properties"; - private static final String METRICS_LOCATION_KEY = "metrics-location"; - private static final String METRICS_LOCATION_DEFAULT = "/dev/shm/performanceanalyzer/"; - private static final String DELETION_INTERVAL_KEY = "metrics-deletion-interval"; - private static final int DELETION_INTERVAL_DEFAULT = 1; - private static final int DELETION_INTERVAL_MIN = 1; - private static final int DELETION_INTERVAL_MAX = 60; - private static final String HTTPS_ENABLED = "https-enabled"; - - /** - * Determines whether the metricsdb files should be cleaned up. - */ - public static final String DB_FILE_CLEANUP_CONF_NAME = "cleanup-metrics-db-files"; - - private String metricsLocation; - private int metricsDeletionInterval; - - /** - * If set to true, the metricsdb files are cleaned up, or else the on-disk files are left out. - */ - private boolean shouldCleanupMetricsDBFiles; - private boolean httpsEnabled; - private Properties settings; - private final String configFilePath; - - static { - PerformanceAnalyzerPlugin.invokePrivilegedAndLogError(() -> createInstance()); - } - - public String getMetricsLocation() { - return metricsLocation; - } - - public int getMetricsDeletionInterval() { - return metricsDeletionInterval * 60 * 1000; - } - - public String getSettingValue(String settingName) { - return settings.getProperty(settingName); - } - - public String getSettingValue(String settingName, String defaultValue) { - return settings.getProperty(settingName, defaultValue); - } - - private void loadHttpsEnabled() throws Exception { - String httpsEnabledString = settings.getProperty(HTTPS_ENABLED, "False"); - if (httpsEnabledString == null) { - httpsEnabled = false; - } - try { - httpsEnabled = Boolean.parseBoolean(httpsEnabledString); - } catch (Exception ex) { - LOG.error("Unable to parse httpsEnabled property with value {}", httpsEnabledString); - httpsEnabled = false; - } - } - - public boolean getHttpsEnabled() { - return this.httpsEnabled; - } - - public boolean shouldCleanupMetricsDBFiles() { - return shouldCleanupMetricsDBFiles; - } - - private PluginSettings(String cfPath) { - metricsLocation = METRICS_LOCATION_DEFAULT; - metricsDeletionInterval = DELETION_INTERVAL_DEFAULT; - if (cfPath == null || cfPath.isEmpty()) { - this.configFilePath = DEFAULT_CONFIG_FILE_PATH; - } else { - this.configFilePath = cfPath; - } - - settings = new Properties(); - try { - settings = getSettingsFromFile(this.configFilePath); - loadMetricsDeletionIntervalFromConfig(); - loadMetricsLocationFromConfig(); - loadHttpsEnabled(); - loadMetricsDBFilesCleanupEnabled(); - } catch (ConfigFileException e) { - LOG.error("Loading config file {} failed with error: {}. Using default values.", - this.configFilePath, e.toString()); - } catch (ConfigFatalException e) { - LOG.error("Having issue to load all config items. Disabling plugin.", e); - ConfigStatus.INSTANCE.setConfigurationInvalid(); - } catch (Exception e) { - LOG.error("Unexpected exception while initializing config. Disabling plugin.", e); - ConfigStatus.INSTANCE.setConfigurationInvalid(); - } - LOG.info("Config: metricsLocation: {}, metricsDeletionInterval: {}, httpsEnabled: {}," + - " cleanup-metrics-db-files: {}", - metricsLocation, metricsDeletionInterval, httpsEnabled, shouldCleanupMetricsDBFiles); - } - - public static PluginSettings instance() { - return instance; - } - - private static void createInstance() { - String cfPath = System.getProperty("configFilePath"); - instance = new PluginSettings(cfPath); - } - - private static Properties getSettingsFromFile(String filePath) throws ConfigFileException { - try { - return SettingsHelper.getSettings(filePath); - } catch (Exception e) { - throw new ConfigFileException(e); - } - } - - private void loadMetricsLocationFromConfig() - throws ConfigFatalException { - if (!settings.containsKey(METRICS_LOCATION_KEY)) { - LOG.info("Cannot find metrics-location, using default value. {}", METRICS_LOCATION_DEFAULT); - } - - metricsLocation = settings.getProperty(METRICS_LOCATION_KEY, METRICS_LOCATION_DEFAULT); - validateOrCreateDir(metricsLocation); - } - - private static void validateOrCreateDir(String path) throws ConfigFatalException { - File dict = new File(path); - - boolean dictCreated = true; - if (!dict.exists()) { - dictCreated = dict.mkdir(); - LOG.info("Trying to create directory {}.", path); - } - - boolean valid = dictCreated && dict.isDirectory() && dict.canWrite(); - if (!valid) { - LOG.error("Invalid metrics location {}." + - " Created: {} (Expect True), Directory: {} (Expect True)," + - " CanWrite: {} (Expect True)", - path, dict.exists(), dict.isDirectory(), dict.canWrite()); - throw new ConfigFatalException("Having issue to use path: " + path); - } - } - - private void loadMetricsDeletionIntervalFromConfig() { - if (!settings.containsKey(DELETION_INTERVAL_KEY)) { - return; - } - - try { - int interval = Integer.parseInt(settings.getProperty(DELETION_INTERVAL_KEY)); - if (interval < DELETION_INTERVAL_MIN || interval > DELETION_INTERVAL_MAX) { - LOG.error("metrics-deletion-interval out of range. Value should in ({}-{}). Using default value {}.", - DELETION_INTERVAL_MIN, DELETION_INTERVAL_MAX, metricsDeletionInterval); - return; - } - metricsDeletionInterval = interval; - } catch (NumberFormatException e) { - LOG.error( - (Supplier) () -> new ParameterizedMessage( - "Invalid metrics-deletion-interval. Using default value {}.", - metricsDeletionInterval), - e); - } - } - private void loadMetricsDBFilesCleanupEnabled() { - String cleanupEnabledString = settings.getProperty(DB_FILE_CLEANUP_CONF_NAME, "True"); - try { - shouldCleanupMetricsDBFiles = Boolean.parseBoolean(cleanupEnabledString); - } catch (Exception ex) { - LOG.error("Unable to parse {} property with value {}. Only true/false expected.", - DB_FILE_CLEANUP_CONF_NAME, cleanupEnabledString); - - // In case of exception, we go with the safe default that the files will always be cleaned up. - shouldCleanupMetricsDBFiles = true; - } - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/SettingsHelper.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/SettingsHelper.java deleted file mode 100644 index 6d9292a1..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/SettingsHelper.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.config; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; - -import java.io.InputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.Properties; - -public class SettingsHelper { - public static Properties getSettings(final String fileRelativePath) throws IOException { - Properties prop = new Properties(); - - try (InputStream input = new FileInputStream(ESResources.INSTANCE.getPluginFileLocation() + fileRelativePath); ) { - // load a properties file - prop.load(input); - } - - return prop; - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/TroubleshootingConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/TroubleshootingConfig.java deleted file mode 100644 index 9b57bb52..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/TroubleshootingConfig.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.config; - -public class TroubleshootingConfig { - public static final boolean enableDevAssert = false; - - public static final boolean getEnableDevAssert() { - return enableDevAssert; - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/config/PerformanceAnalyzerConfigAction.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/config/PerformanceAnalyzerConfigAction.java index 1dfa37df..42d1ed6f 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/config/PerformanceAnalyzerConfigAction.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/http_action/config/PerformanceAnalyzerConfigAction.java @@ -16,6 +16,8 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.config; import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerPlugin; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ScheduledMetricCollectorsExecutor; + import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; @@ -50,6 +52,7 @@ public class PerformanceAnalyzerConfigAction extends BaseRestHandler { private static PerformanceAnalyzerConfigAction instance = null; private boolean isInitialized = false; private boolean featureEanbledDefaultValue = true; + private ScheduledMetricCollectorsExecutor scheduledMetricCollectorsExecutor; public static PerformanceAnalyzerConfigAction getInstance() { return instance; @@ -61,10 +64,12 @@ public static void setInstance(PerformanceAnalyzerConfigAction performanceanalyz private static final String METRIC_ENABLED_CONF_FILENAME = "performance_analyzer_enabled.conf"; @Inject - public PerformanceAnalyzerConfigAction(Settings settings, RestController controller) { + public PerformanceAnalyzerConfigAction(Settings settings, RestController controller, + ScheduledMetricCollectorsExecutor scheduledMetricCollectorsExecutor) { super(settings); controller.registerHandler(org.elasticsearch.rest.RestRequest.Method.GET, "/_opendistro/_performanceanalyzer/config", this); controller.registerHandler(org.elasticsearch.rest.RestRequest.Method.POST, "/_opendistro/_performanceanalyzer/config", this); + this.scheduledMetricCollectorsExecutor = scheduledMetricCollectorsExecutor ; this.featureEnabled = getFeatureEnabledFromConf(); LOG.info("PerformanceAnalyzer Enabled: {}", this.featureEnabled); } @@ -82,6 +87,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli bValue, this.featureEnabled); if (this.featureEnabled != bValue) { this.featureEnabled = (Boolean) value; + if( scheduledMetricCollectorsExecutor != null) { + scheduledMetricCollectorsExecutor.setEnabled(this.featureEnabled); + } saveFeatureEnabledToConf(this.featureEnabled); } } @@ -138,10 +146,16 @@ private boolean getFeatureEnabledFromConf() { String nextLine = sc.nextLine(); featureEnabled = Boolean.parseBoolean(nextLine); isInitialized = true; + if ( scheduledMetricCollectorsExecutor != null) { + scheduledMetricCollectorsExecutor.setEnabled(featureEnabled); + } } catch (java.nio.file.NoSuchFileException ex) { saveFeatureEnabledToConf(featureEanbledDefaultValue); isInitialized = true; featureEnabled = featureEanbledDefaultValue; + if( scheduledMetricCollectorsExecutor != null) { + scheduledMetricCollectorsExecutor.setEnabled(featureEnabled); + } } catch (Exception e) { LOG.error("Error reading Feature Enabled from Conf file", e); featureEnabled = featureEanbledDefaultValue; diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/hwnet/Disks.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/hwnet/Disks.java deleted file mode 100644 index a714c96b..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/hwnet/Disks.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.hwnet; - -import java.io.File; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerPlugin; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.DiskMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux.LinuxDiskMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.SchemaFileParser; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.DiskMetrics; - -public class Disks { - private static Map> diskKVMap = new HashMap<>(); - private static Map> olddiskKVMap = new HashMap<>(); - private static long kvTimestamp = 0; - private static long oldkvTimestamp = 0; - private static Set diskList = new HashSet<>(); - private static final Logger LOG = LogManager.getLogger(Disks.class); - private static LinuxDiskMetricsGenerator linuxDiskMetricsHandler = new LinuxDiskMetricsGenerator(); - - private static String statKeys[] = { - "majno", //1 - "minno", - "name", - "rdone", - "rmerged", - "rsectors", - "rtime", - "wdone", - "wmerged", - "wsectors", //10 - "wtime", - "inprogressIO", - "IOtime", - "weightedIOtime" - }; - - private static SchemaFileParser.FieldTypes statTypes[] = { - SchemaFileParser.FieldTypes.INT, //1 - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.STRING, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, //10 - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG - }; - - static { - PerformanceAnalyzerPlugin.invokePrivileged(() -> listDisks()); - oldkvTimestamp = System.currentTimeMillis(); - kvTimestamp = oldkvTimestamp; - } - - private static StringBuilder value = new StringBuilder(); - - private static void listDisks() { - try { - File file = new File("/sys/block"); - for (File dfile : file.listFiles()) { - if (!dfile.getCanonicalPath().contains("/virtual/")) { - diskList.add(dfile.getName()); - } - } - } catch(Exception e) { - LOG.debug("Exception in calling listDisks with details: {}", () -> e.toString()); - } - } - - public static DiskMetricsGenerator getDiskMetricsHandler() { - return linuxDiskMetricsHandler; - } - - public static void addSample() { - olddiskKVMap.clear(); - olddiskKVMap.putAll(diskKVMap); - diskKVMap.clear(); - - SchemaFileParser parser = - new SchemaFileParser("/proc/diskstats", statKeys, statTypes); - List> sampleList = parser.parseMultiple(); - - for (Map sample: sampleList) { - String diskname = (String)(sample.get("name")); - if (!diskList.contains(diskname)) { - diskKVMap.put(diskname, sample); - } - } - - oldkvTimestamp = kvTimestamp; - kvTimestamp = System.currentTimeMillis(); - - calculateDiskMetrics(); - } - - private static void calculateDiskMetrics() { - - linuxDiskMetricsHandler.setDiskMetricsMap(getMetricsMap()); - } - - public static Map getMetricsMap() { - Map map = new HashMap<>(); - if (kvTimestamp > oldkvTimestamp) { - for (String disk : diskKVMap.keySet()) { - Map m = diskKVMap.get(disk); - Map mold = olddiskKVMap.get(disk); - if (mold != null) { - DiskMetrics dm = new DiskMetrics(); - dm.name = (String) m.get("name"); - double rwdeltatime = 1.0 * ((long) m.get("rtime") + (long) m.get("wtime") - - (long) mold.get("rtime") - (long) mold.get("wtime")); - double rwdeltaiops = 1.0 * ((long) m.get("rdone") + (long) m.get("wdone") - - (long) mold.get("rdone") - (long) mold.get("wdone")); - double rwdeltasectors = 1.0 * ((long) m.get("rsectors") + (long) m.get("wsectors") - - (long) mold.get("rsectors") - (long) mold.get("wsectors")); - - dm.utilization = rwdeltatime / (kvTimestamp - oldkvTimestamp); - dm.await = (rwdeltaiops > 0) ? rwdeltatime / rwdeltaiops : 0; - dm.serviceRate = (rwdeltatime > 0) ? rwdeltasectors * 512 * 1.0e-3 / rwdeltatime : 0; - - map.put(disk, dm); - } - } - } - return map; - } - - public static void runOnce() { - addSample(); - System.out.println("disks: "+ getMetricsMap()); - } -} - - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/hwnet/NetworkE2E.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/hwnet/NetworkE2E.java deleted file mode 100644 index bfb8986a..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/hwnet/NetworkE2E.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.hwnet; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux.LinuxTCPMetricsGenerator; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.OSGlobals; -import com.google.common.annotations.VisibleForTesting; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; - -public class NetworkE2E { - /* Data sources: - /proc/net/tcp, /proc/net/tcp6 and /proc/pid/fd/* - intersection of these gives a list of flows - owned by the process. net/tcp gives metrics - (by src-dest pair) around queues, retx's - and TCP sndwnd. - */ - - private static final Logger LOG = LogManager.getLogger( - NetworkE2E.class); - private static String pid = OSGlobals.getPid(); - - static class TCPFlowMetrics { - String destIP; - - long txQueue; - long rxQueue; - long currentLost; - long sendCWND; - long SSThresh; - } - - static class destTCPFlowMetrics { - long txQueueTot; - long rxQueueTot; - long currentLostTot; - long sendCWNDTot; - long SSThreshTot; - int numFlows; - destTCPFlowMetrics(TCPFlowMetrics m) { - txQueueTot = m.txQueue; - rxQueueTot = m.rxQueue; - currentLostTot = m.currentLost; - sendCWNDTot = m.sendCWND; - SSThreshTot = m.SSThresh; - numFlows = 1; - } - } - - private static Set inodeSocketList - = new HashSet<>(); - private static Map inodeFlowMetricsMap - = new HashMap<>(); - private static Map destnodeFlowMetricsMap - = new HashMap<>(); - private static LinuxTCPMetricsGenerator linuxTCPMetricsHandler = new LinuxTCPMetricsGenerator(); - - private static StringBuilder value = new StringBuilder(); - - static void listSockets() { - File self = new File("/proc/" + pid + "/fd"); - File[] filesList = self.listFiles(); - for (File f : filesList) { - // no check for file, as this dir is all files/symlinks - String target = null; - try { - Path targetp = Files.readSymbolicLink(Paths.get(f.getCanonicalPath())); - target = targetp.toString(); - } catch (Exception e) { - continue; - } - if (target.contains("socket:")) { - target = target.split("socket:\\[")[1]; - target = target.split("\\]")[0]; - inodeSocketList.add(target); - } - } - } - - private static void generateMap(String line, String ver) { - String[] toks = line.trim().split("\\s+"); - if (!inodeSocketList.contains(toks[9])) { // inode - return; - } - TCPFlowMetrics m = new TCPFlowMetrics(); - m.destIP = toks[2].split(":")[0]; - m.txQueue = Long.decode("0x" + toks[4].split(":")[0]); - m.rxQueue = Long.decode("0x" + toks[4].split(":")[1]); - m.currentLost = Long.decode("0x" + toks[6]); - if (toks.length > 16) { - m.sendCWND = Long.parseLong(toks[15]); - m.SSThresh = Long.parseLong(toks[16]); - } else { - m.sendCWND = -1; - m.SSThresh = -1; - } - inodeFlowMetricsMap.put(toks[9], m); - } - - private static void mapTCPMetrics(String ver) { - int ln = 0; - try (FileReader fileReader = new FileReader(new File(ver)); - BufferedReader bufferedReader = new BufferedReader(fileReader)) { - String line = null; - while ((line = bufferedReader.readLine()) != null) { - if (ln != 0) { // first line is keys - generateMap(line, ver); - } - ln++; - } - } catch (Exception e) { - LOG.debug("Error in mapTCPMetrics: {} with ExceptionCode: {}", - () -> e, () -> StatExceptionCode.NETWORK_COLLECTION_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.NETWORK_COLLECTION_ERROR); - } - } - - private static void mapTCPMetrics() { - mapTCPMetrics("/proc/net/tcp"); - mapTCPMetrics("/proc/net/tcp6"); - } - - private static void clearAll() { - inodeSocketList.clear(); - inodeFlowMetricsMap.clear(); - destnodeFlowMetricsMap.clear(); - } - - private static void computeSummary() { - for (String inode : inodeFlowMetricsMap.keySet()) { - TCPFlowMetrics m = inodeFlowMetricsMap.get(inode); - destTCPFlowMetrics exist = destnodeFlowMetricsMap.get(m.destIP); - if (exist == null) { - destnodeFlowMetricsMap.put(m.destIP, new destTCPFlowMetrics(m)); - } else { - // check for "-1"s and add to total only if it is not -1 - exist.numFlows++; - exist.txQueueTot += (m.txQueue != -1 ? m.txQueue : 0); - exist.rxQueueTot += (m.rxQueue != -1 ? m.rxQueue : 0); - exist.currentLostTot += (m.currentLost != -1 ? m.currentLost : 0); - exist.sendCWNDTot += (m.sendCWND != -1 ? m.sendCWND : 0); - exist.SSThreshTot += (m.SSThresh != -1 ? m.SSThresh : 0); - } - } - - calculateTCPMetrics(); - } - - protected static void calculateTCPMetrics() { - - Map localMap = new HashMap<>(); - for (String dest : destnodeFlowMetricsMap.keySet()) { - destTCPFlowMetrics m = destnodeFlowMetricsMap.get(dest); - - double[] metrics = new double[6]; - metrics[0] = m.numFlows; - metrics[1] = m.txQueueTot * 1.0 / m.numFlows; - metrics[2] = m.rxQueueTot * 1.0 / m.numFlows; - metrics[3] = m.currentLostTot * 1.0 / m.numFlows; - metrics[4] = m.sendCWNDTot * 1.0 / m.numFlows; - metrics[5] = m.SSThreshTot * 1.0 / m.numFlows; - - localMap.put(dest, metrics); - } - - linuxTCPMetricsHandler.setTCPMetrics(localMap); - } - - public static LinuxTCPMetricsGenerator getTCPMetricsHandler() { - - return linuxTCPMetricsHandler; - } - - public static void addSample() { - clearAll(); - listSockets(); - mapTCPMetrics(); - computeSummary(); - } - - public static void runOnce() { - clearAll(); - listSockets(); - mapTCPMetrics(); - computeSummary(); - } - - @VisibleForTesting - protected static void setDestnodeFlowMetricsMap( - Map destnodeFlowMetricsMap) { - NetworkE2E.destnodeFlowMetricsMap = destnodeFlowMetricsMap; - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/hwnet/NetworkInterface.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/hwnet/NetworkInterface.java deleted file mode 100644 index 8b5a3304..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/hwnet/NetworkInterface.java +++ /dev/null @@ -1,378 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.hwnet; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.util.HashMap; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NetInterfaceSummary; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux.LinuxIPMetricsGenerator; -import com.google.common.annotations.VisibleForTesting; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; - -public class NetworkInterface { - private static final Logger LOG = LogManager.getLogger(NetworkInterface.class); - - /* Data sources: - /proc/net/snmp, /prov/net/snmp6, /proc/net/dev - measures tcp and ip-layer pathologies. - SNMP fields of interest (see RFCs 2011 and 1213): - - [ip6]inReceives: total including errors - - [ip6]inDelivers: sent to next layer (including ICMP) - - [ip6]outRequests: sent from previous layer - - [ip6]outDiscards + [ip6]outNoRoutes: sender-side drops - */ - - static class NetInterfaceMetrics { - Map PHYmetrics = - new HashMap<>(); - Map IPmetrics = - new HashMap<>(); - // these three are currently unused; - // leaving them commented for now. - /*Map TCPmetrics = - new HashMap<>(); - Map UDPmetrics = - new HashMap<>(); - Map ICMPmetrics = - new HashMap<>();*/ - public void clearAll() { - PHYmetrics.clear(); - IPmetrics.clear(); - /*TCPmetrics.clear(); - UDPmetrics.clear(); - ICMPmetrics.clear();*/ - } - public void putAll(NetInterfaceMetrics m) { - PHYmetrics.putAll(m.PHYmetrics); - IPmetrics.putAll(m.IPmetrics); - /*TCPmetrics.putAll(m.TCPmetrics); - UDPmetrics.putAll(m.UDPmetrics); - ICMPmetrics.putAll(m.ICMPmetrics);*/ - } - } - private static NetInterfaceMetrics currentMetrics = new NetInterfaceMetrics(); - private static NetInterfaceMetrics oldMetrics = new NetInterfaceMetrics(); - private static Map currentMetrics6 = new HashMap<>(); - private static Map oldMetrics6 = new HashMap<>(); - private static long kvTimestamp = 0; - private static long oldkvTimestamp = 0; - - private static StringBuilder ret = new StringBuilder(); - - private static String[] IPkeys = null; -// static private String[] TCPkeys = null; -// static private String[] UDPkeys = null; -// static private String[] ICMPkeys = null; - - private static LinuxIPMetricsGenerator linuxIPMetricsGenerator = new LinuxIPMetricsGenerator(); - - static { - addSampleHelper(); - } - - public static LinuxIPMetricsGenerator getLinuxIPMetricsGenerator() { - return linuxIPMetricsGenerator; - } - - - protected static void calculateNetworkMetrics() { - - if (kvTimestamp <= oldkvTimestamp) { - linuxIPMetricsGenerator.setInNetworkInterfaceSummary(null); - linuxIPMetricsGenerator.setOutNetworkInterfaceSummary(null); - return; - } - - Map curphy = currentMetrics.PHYmetrics; - Map curipv4 = currentMetrics.IPmetrics; - Map oldphy = oldMetrics.PHYmetrics; - Map oldipv4 = oldMetrics.IPmetrics; - - long nin = curipv4.get("InReceives") - oldipv4.get("InReceives"); - long nout = curipv4.get("OutRequests") - oldipv4.get("OutRequests"); - long delivin = curipv4.get("InDelivers") - oldipv4.get("InDelivers"); - long dropout = curipv4.get("OutDiscards") + curipv4.get("OutNoRoutes") - - oldipv4.get("OutDiscards") - oldipv4.get("OutNoRoutes"); - long nin6 = currentMetrics6.get("Ip6InReceives") - oldMetrics6.get("Ip6InReceives"); - long nout6 = currentMetrics6.get("Ip6OutRequests") - oldMetrics6.get("Ip6OutRequests"); - long delivin6 = currentMetrics6.get("Ip6InDelivers") - oldMetrics6.get("Ip6InDelivers"); - long dropout6 = currentMetrics6.get("Ip6OutDiscards") + currentMetrics6.get("Ip6OutNoRoutes") - - oldMetrics6.get("Ip6OutDiscards") - oldMetrics6.get("Ip6OutNoRoutes"); - - long timeDelta = kvTimestamp - oldkvTimestamp; - double inbps = 8*1.0e3*(curphy.get("inbytes") - oldphy.get("inbytes")) / timeDelta; - double outbps = 8*1.0e3*(curphy.get("outbytes") - oldphy.get("outbytes")) / timeDelta; - double inPacketRate4 = 1.0e3*(nin) / timeDelta; - double outPacketRate4 = 1.0e3*(nout) / timeDelta; - double inDropRate4 = 1.0e3*(nin - delivin) / timeDelta; - double outDropRate4 = 1.0e3*(dropout) / timeDelta; - double inPacketRate6 = 1.0e3*(nin6) / timeDelta; - double outPacketRate6 = 1.0e3*(nout6) / timeDelta; - double inDropRate6 = 1.0e3*(nin6 - delivin6) / timeDelta; - double outDropRate6 = 1.0e3*(dropout6) / timeDelta; - - NetInterfaceSummary inNetwork = new NetInterfaceSummary( - NetInterfaceSummary.Direction.in, - inPacketRate4, - inDropRate4, - inPacketRate6, - inDropRate6, - inbps - ); - - NetInterfaceSummary outNetwork = new NetInterfaceSummary( - NetInterfaceSummary.Direction.out, - outPacketRate4, - outDropRate4, - outPacketRate6, - outDropRate6, - outbps - ); - - linuxIPMetricsGenerator.setInNetworkInterfaceSummary(inNetwork); - linuxIPMetricsGenerator.setOutNetworkInterfaceSummary(outNetwork); - } - - private static void getKeys(String line) { - if (IPkeys != null) { - //{ && TCPkeys != null && - //UDPkeys != null && ICMPkeys != null) { - return; - } - if (line.startsWith("Ip:")) { - IPkeys = line.split("\\s+"); - } /*else if (line.startsWith("Icmp:")) { - ICMPkeys = line.split("\\s+"); - } else if (line.startsWith("Tcp:")) { - TCPkeys = line.split("\\s+"); - } else if (line.startsWith("Udp:")) { - UDPkeys = line.split("\\s+"); - }*/ - } - - private static void generateMap(String line) { - Map map = null; - String[] keys = null; - if (line.startsWith("Ip:")) { - map = currentMetrics.IPmetrics; - keys = IPkeys; - } /*else if (line.startsWith("Icmp:")) { - map = currentMetrics.ICMPmetrics; - keys = ICMPkeys; - } else if (line.startsWith("Tcp:")) { - map = currentMetrics.TCPmetrics; - keys = TCPkeys; - } else if (line.startsWith("Udp:")) { - map = currentMetrics.UDPmetrics; - keys = UDPkeys; - }*/ - if (keys != null) { - generateMap(line, keys, map); - } - } - - private static void generateMap(String line, String[] keys, Map map) { - String[] values = line.split("\\s+"); - int count = values.length; - map.put(keys[0], 0L); - for (int i = 1; i < count; i++) { - map.put(keys[i], Long.parseLong(values[i])); - } - } - - private static void addSample4() { - int ln = 0; - - oldMetrics.clearAll(); - oldMetrics.putAll(currentMetrics); - currentMetrics.clearAll(); - oldkvTimestamp = kvTimestamp; - kvTimestamp = System.currentTimeMillis(); - - try (FileReader fileReader = new FileReader(new File("/proc/net/snmp")); - BufferedReader bufferedReader = new BufferedReader(fileReader);) { - String line = null; - while ((line = bufferedReader.readLine()) != null) { - if (ln % 2 == 0) { // keys - getKeys(line); - } else { - generateMap(line); - } - ln++; - } - } catch (Exception e) { - LOG.debug("Exception in calling addSample4 with details: {} with ExceptionCode: {}", - () -> e.toString(), () -> StatExceptionCode.NETWORK_COLLECTION_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.NETWORK_COLLECTION_ERROR); - } - } - - private static void addSample6() { - oldMetrics6.clear(); - oldMetrics6.putAll(currentMetrics6); - currentMetrics6.clear(); - - try (FileReader fileReader = new FileReader(new File("/proc/net/snmp6")); - BufferedReader bufferedReader = new BufferedReader(fileReader);) { - String line = null; - while ((line = bufferedReader.readLine()) != null) { - String[] toks = line.split("[ \\t]+"); - if (toks.length > 1) { - currentMetrics6.put(toks[0], Long.parseLong(toks[1])); - } - } - } catch (Exception e) { - LOG.debug("Exception in calling addSample6 with details: {} with ExceptionCode: {}", - () -> e.toString(), () -> StatExceptionCode.NETWORK_COLLECTION_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.NETWORK_COLLECTION_ERROR); - } - } - - // this assumes that addSample4() is called - private static void addDeviceStats() { - try (FileReader fileReader = new FileReader(new File("/proc/net/dev")); - BufferedReader bufferedReader = new BufferedReader(fileReader);) { - String line = null; - long intotbytes = 0, outtotbytes = 0; - long intotpackets = 0, outtotpackets = 0; - while ((line = bufferedReader.readLine()) != null) { - if (line.contains("Receive") || line.contains("packets")) { - continue; - } - String[] toks = line.trim().split(" +"); - intotbytes += Long.parseLong(toks[1]); - intotpackets += Long.parseLong(toks[2]); - outtotbytes += Long.parseLong(toks[9]); - outtotpackets += Long.parseLong(toks[10]); - } - currentMetrics.PHYmetrics.put("inbytes", intotbytes); - currentMetrics.PHYmetrics.put("inpackets", intotpackets); - currentMetrics.PHYmetrics.put("outbytes", outtotbytes); - currentMetrics.PHYmetrics.put("outpackets", outtotpackets); - } catch (Exception e) { - LOG.debug("Exception in calling addDeviceStats with details: {} with ExceptionCode: {}", - () -> e.toString(), () -> StatExceptionCode.NETWORK_COLLECTION_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.NETWORK_COLLECTION_ERROR); - } - } - - public static void addSample() { - addSampleHelper(); - calculateNetworkMetrics(); - } - - private static synchronized void addSampleHelper() { - addSample4(); - addSample6(); - addDeviceStats(); - } - - public static void runOnce() { - addSample(); - } - - @VisibleForTesting - Map getCurrentPhyMetric() { - return currentMetrics.PHYmetrics; - } - - @VisibleForTesting - Map getCurrentIpMetric() { - return currentMetrics.IPmetrics; - } - - @VisibleForTesting - Map getOldPhyMetric() { - return oldMetrics.PHYmetrics; - } - - @VisibleForTesting - Map getOldIpMetric() { - return oldMetrics.IPmetrics; - } - - @VisibleForTesting - Map getCurrentMetrics6() { - return currentMetrics6; - } - - @VisibleForTesting - Map getOldMetrics6() { - return oldMetrics6; - } - - @VisibleForTesting - void putCurrentPhyMetric(String key, Long value) { - currentMetrics.PHYmetrics.put(key, value); - } - - @VisibleForTesting - void putCurrentIpMetric(String key, Long value) { - currentMetrics.IPmetrics.put(key, value); - } - - @VisibleForTesting - void putOldPhyMetric(String key, Long value) { - oldMetrics.PHYmetrics.put(key, value); - } - - @VisibleForTesting - void putOldIpMetric(String key, Long value) { - oldMetrics.IPmetrics.put(key, value); - } - - @VisibleForTesting - void putCurrentMetrics6(String key, Long value) { - currentMetrics6.put(key, value); - } - - @VisibleForTesting - void putOldMetrics6(String key, Long value) { - oldMetrics6.put(key, value); - } - - @VisibleForTesting - static void setKvTimestamp(long value) { - NetworkInterface.kvTimestamp = value; - } - - @VisibleForTesting - static void setOldkvTimestamp(long oldkvTimestamp) { - NetworkInterface.oldkvTimestamp = oldkvTimestamp; - } - - @VisibleForTesting - static long getKvTimestamp() { - return kvTimestamp; - } - - @VisibleForTesting - static long getOldkvTimestamp() { - return oldkvTimestamp; - } - -} - - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/GCMetrics.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/GCMetrics.java deleted file mode 100644 index a086c8ad..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/GCMetrics.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm; - -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class GCMetrics { - private static GarbageCollectorMXBean fullGC = null; - private static GarbageCollectorMXBean youngGC = null; - - private static long totYoungGCCollectionCount = 0; - private static long totYoungGCCollectionTime = 0; - private static long totFullGCCollectionCount = 0; - private static long totFullGCCollectionTime = 0; - - private static long lastYoungGCCollectionCount = 0; - private static long lastYoungGCCollectionTime = 0; - private static long lastFullGCCollectionCount = 0; - private static long lastFullGCCollectionTime = 0; - private static final Logger LOGGER = LogManager.getLogger(GCMetrics.class); - - static { - for (GarbageCollectorMXBean item : ManagementFactory.getGarbageCollectorMXBeans()) { - if ("ConcurrentMarkSweep".equals(item.getName()) - || "MarkSweepCompact".equals(item.getName()) - || "PS MarkSweep".equals(item.getName()) - || "G1 Old Generation".equals(item.getName()) - || "Garbage collection optimized for short pausetimes Old Collector".equals(item.getName()) - || "Garbage collection optimized for throughput Old Collector".equals(item.getName()) - || "Garbage collection optimized for deterministic pausetimes Old Collector".equals(item.getName()) - ) { - fullGC = item; - } else if ("ParNew".equals(item.getName()) - || "Copy".equals(item.getName()) - || "PS Scavenge".equals(item.getName()) - || "G1 Young Generation".equals(item.getName()) - || "Garbage collection optimized for short pausetimes Young Collector".equals(item.getName()) - || "Garbage collection optimized for throughput Young Collector".equals(item.getName()) - || "Garbage collection optimized for deterministic pausetimes Young Collector".equals(item.getName()) - ) { - youngGC = item; - } else { - LOGGER.error("MX bean missing: {}", () -> item.getName()); - } - } - } - - public static long getTotYoungGCCollectionCount() { - return totYoungGCCollectionCount; - } - - public static long getTotYoungGCCollectionTime() { - return totYoungGCCollectionTime; - } - - public static long getTotFullGCCollectionCount() { - return totFullGCCollectionCount; - } - - public static long getTotFullGCCollectionTime() { - return totFullGCCollectionTime; - } - - private static long getYoungGCCollectionCount() { - if (youngGC == null) { - return 0; - } - return youngGC.getCollectionCount(); - } - - private static long getYoungGCCollectionTime() { - if (youngGC == null) { - return 0; - } - return youngGC.getCollectionTime(); - } - - private static long getFullGCCollectionCount() { - if (fullGC == null) { - return 0; - } - return fullGC.getCollectionCount(); - } - - private static long getFullGCCollectionTime() { - if (fullGC == null) { - return 0; - } - return fullGC.getCollectionTime(); - } - - public static void runGCMetrics() { - long YoungGCCollectionCount = getYoungGCCollectionCount(); - long YoungGCCollectionTime = getYoungGCCollectionTime(); - long FullGCCollectionCount = getFullGCCollectionCount(); - long FullGCCollectionTime = getFullGCCollectionTime(); - - totYoungGCCollectionCount = - YoungGCCollectionCount - lastYoungGCCollectionCount; - totYoungGCCollectionTime = - YoungGCCollectionTime - lastYoungGCCollectionTime; - totFullGCCollectionCount = - FullGCCollectionCount - lastFullGCCollectionCount; - totFullGCCollectionTime = - FullGCCollectionTime - lastFullGCCollectionTime; - - lastYoungGCCollectionCount = YoungGCCollectionCount; - lastYoungGCCollectionTime = YoungGCCollectionTime; - lastFullGCCollectionCount = FullGCCollectionCount; - lastFullGCCollectionTime = FullGCCollectionTime; - } - - static void printGCMetrics() { - if (lastYoungGCCollectionCount >= 0) { - System.out.println("GC:: yC:" + getTotYoungGCCollectionCount() + - " yT:" + getTotYoungGCCollectionTime() + - " oC:" + getTotFullGCCollectionCount() + - " oT:" + getTotFullGCCollectionTime()); - } - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/HeapMetrics.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/HeapMetrics.java deleted file mode 100644 index 9197665c..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/HeapMetrics.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm; - -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.lang.management.MemoryPoolMXBean; -import java.lang.management.MemoryUsage; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; - -public class HeapMetrics { - private static final Map> memoryUsageSuppliers; - - static { - memoryUsageSuppliers = new HashMap<>(); - MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); - if (memoryMXBean != null) { - memoryUsageSuppliers.put("Heap", () -> memoryMXBean.getHeapMemoryUsage()); - memoryUsageSuppliers.put("NonHeap", () -> memoryMXBean.getNonHeapMemoryUsage()); - } - - List list = ManagementFactory.getMemoryPoolMXBeans(); - for (MemoryPoolMXBean item : list) { - if ("CMS Perm Gen".equals(item.getName()) - || "Perm Gen".equals(item.getName()) - || "PS Perm Gen".equals(item.getName()) - || "G1 Perm Gen".equals(item.getName()) - || "Metaspace".equals(item.getName())) { - memoryUsageSuppliers.put("PermGen", () -> item.getUsage()); - } else if ("CMS Old Gen".equals(item.getName()) - || "Tenured Gen".equals(item.getName()) - || "PS Old Gen".equals(item.getName()) - || "G1 Old Gen".equals(item.getName())) { - memoryUsageSuppliers.put("OldGen", () -> item.getUsage()); - } else if ("Par Eden Space".equals(item.getName()) - || "Eden Space".equals(item.getName()) - || "PS Eden Space".equals(item.getName()) - || "G1 Eden".equals(item.getName())) { - memoryUsageSuppliers.put("Eden", () -> item.getUsage()); - } else if ("Par Survivor Space".equals(item.getName()) - || "Survivor Space".equals(item.getName()) - || "PS Survivor Space".equals(item.getName()) - || "G1 Survivor".equals(item.getName())) { - memoryUsageSuppliers.put("Survivor", () -> item.getUsage()); - } - } - } - - public static Map> getMemoryUsageSuppliers() { - return memoryUsageSuppliers; - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/ThreadList.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/ThreadList.java deleted file mode 100644 index ba9d94e1..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/ThreadList.java +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.management.ManagementFactory; -import java.lang.management.ThreadInfo; -import java.lang.management.ThreadMXBean; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerPlugin; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.OSMetricsGeneratorFactory; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.sun.tools.attach.VirtualMachine; - -import sun.tools.attach.HotSpotVirtualMachine; - - -/** Traverses and prints the stack traces for all Java threads in the - * remote VM */ -public class ThreadList { - private static final Map jTidNameMap = new ConcurrentHashMap<>(); - private static final Map nativeTidMap = new ConcurrentHashMap<>(); - private static final Map oldNativeTidMap = new ConcurrentHashMap<>(); - private static final Map jTidMap = new ConcurrentHashMap<>(); - private static final Map nameMap = new ConcurrentHashMap<>(); - private static final String pid = OSMetricsGeneratorFactory.getInstance().getPid(); - static final Logger LOGGER = LogManager.getLogger(ThreadList.class); - static final int samplingInterval = MetricsConfiguration.CONFIG_MAP.get(ThreadList.class).samplingInterval; - private static final long minRunInterval = samplingInterval; - private static final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); - private static final Pattern linePattern = Pattern.compile("\"([^\"]*)\""); - private static long lastRunTime = 0; - - public static class ThreadState { - public long javaTid; - public long nativeTid; - public long heapUsage; - public String threadName; - public String tState; - public Thread.State state; - public long blockedCount; - public long blockedTime; - - public double heapAllocRate; - public double avgBlockedTime; - - ThreadState() { - javaTid = -1; - nativeTid = -1; - heapUsage = -1; - heapAllocRate = 0; - blockedCount = 0; - blockedTime = 0; - avgBlockedTime = 0; - threadName = ""; - tState = ""; - } - - @Override - public String toString() { - return new StringBuilder().append("javatid:").append(javaTid).append(" nativetid:") - .append(nativeTid).append(" name:").append(threadName).append(" state:") - .append(tState).append("(").append(state).append(")").append(" heaprate: ").append(heapAllocRate) - .append(" bTime: ").append(avgBlockedTime).append(":").append(blockedCount).toString(); - } - } - - public static Map getNativeTidMap() { - synchronized (ThreadList.class) { - if (System.currentTimeMillis() > lastRunTime + minRunInterval) { - runThreadDump(pid, new String[0]); - } - //- sending a copy so that if runThreadDump next iteration clears it; caller still has the state at the call time - //- not too expensive as this is only being called from Scheduled Collectors (only once in few seconds) - return new HashMap<>(nativeTidMap); - } - } - - - public static ThreadState getThreadState(long threadId) { - ThreadState retVal = jTidMap.get(threadId); - - if (retVal != null) { - return retVal; - } - - synchronized (ThreadList.class) { - retVal = jTidMap.get(threadId); - - if (retVal != null) { - return retVal; - } - - runThreadDump(pid, new String[0]); - } - - return jTidMap.get(threadId); - } - - // Attach to pid and perform a thread dump - private static void runAttachDump(String pid, String[] args) { - VirtualMachine vm = null; - try { - vm = VirtualMachine.attach(pid); - } catch (Exception ex) { - LOGGER.debug("Error in Attaching to VM with exception: {} with ExceptionCode: {}", - () -> ex.toString(), () -> StatExceptionCode.JVM_ATTACH_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.JVM_ATTACH_ERROR); - return; - } - - try (InputStream in = ((HotSpotVirtualMachine) vm).remoteDataDump((Object[]) args);) { - createMap(in); - } catch (Exception ex) { - LOGGER.debug("Cannot list threads with exception: {} with ExceptionCode: {}", - () -> ex.toString(), () -> StatExceptionCode.JVM_ATTACH_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.JVM_ATTACH_ERROR); - } - - try { - vm.detach(); - } catch (Exception ex) { - LOGGER.debug("Failed in VM Detach with exception: {} with ExceptionCode: {}", - () -> ex.toString(), () -> StatExceptionCode.JVM_ATTACH_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.JVM_ATTACH_ERROR); - } - } - - //ThreadMXBean-based info for tid, name and allocs - private static void runMXDump() { - long[] ids = threadBean.getAllThreadIds(); - ThreadInfo[] infos = threadBean.getThreadInfo(ids); - for (ThreadInfo info : infos) { - long id = info.getThreadId(); - String name = info.getThreadName(); - Thread.State state = info.getThreadState(); - - // following captures cumulative allocated bytes + TLAB used bytes - // and it is cumulative - long mem = ((com.sun.management.ThreadMXBean) threadBean).getThreadAllocatedBytes(id); - - ThreadState t = jTidMap.get(id); - if (t == null) { - continue; - } - t.heapUsage = mem; - t.state = state; - t.blockedCount = info.getBlockedCount(); - t.blockedTime = info.getBlockedTime(); - ThreadHistory.add(t.nativeTid, - (state == Thread.State.BLOCKED) ? samplingInterval : 0); - - long curRunTime = System.currentTimeMillis(); - ThreadState oldt = oldNativeTidMap.get(t.nativeTid); - if (curRunTime > lastRunTime && oldt != null) { - t.heapAllocRate = Math.max(t.heapUsage - oldt.heapUsage, 0) * 1.0e3 - /(curRunTime - lastRunTime); - if (t.blockedTime != -1 && t.blockedCount > oldt.blockedCount) { - t.avgBlockedTime = 1.0e-3 * (t.blockedTime - oldt.blockedTime) - / (t.blockedCount - oldt.blockedCount); - } else { - CircularLongArray arr = ThreadHistory.tidHistoryMap.get(t.nativeTid); - // NOTE: this is an upper bound - if (arr != null) { - t.avgBlockedTime = 1.0 * arr.getAvgValue() / samplingInterval; - } - } - } - jTidNameMap.put(id, name); - } - ThreadHistory.cleanup(); - } - - static void runThreadDump(String pid, String[] args) { - jTidNameMap.clear(); - oldNativeTidMap.putAll(nativeTidMap); - nativeTidMap.clear(); - jTidMap.clear(); - nameMap.clear(); - - //TODO: make this map update atomic - PerformanceAnalyzerPlugin.invokePrivileged(() -> runAttachDump(pid, args)); - runMXDump(); - - lastRunTime = System.currentTimeMillis(); - } - - private static void parseLine(String line) { - String[] tokens = line.split(" os_prio=[0-9]* "); - ThreadState t = new ThreadState(); - t.javaTid = -1; - - Matcher m = linePattern.matcher(tokens[0]); - if (!m.find()) { - t.threadName = tokens[0]; - } else { - t.threadName = m.group(1); - if (!tokens[0].equals("\"" + t.threadName + "\"")) { - t.javaTid = Long.parseLong( - tokens[0].split(Pattern.quote("\"" + t.threadName + "\" "))[1]. - split(" ")[0]. - split("#")[1]); - } - } - - tokens = tokens[1].split(" "); - for (String token : tokens) { - String[] keyValuePare = token.split("="); - if (keyValuePare.length < 2) { - continue; - } - if (t.javaTid == -1 && keyValuePare[0].equals("tid")) { - t.javaTid = Long.decode(keyValuePare[1]); - } - if (keyValuePare[0].equals("nid")) { - t.nativeTid = Long.decode(keyValuePare[1]); - } - } - t.tState = tokens[2]; //TODO: stuff like "in Object.wait()" - nativeTidMap.put(t.nativeTid, t); - jTidMap.put(t.javaTid, t); - nameMap.put(t.threadName, t); //XXX: we assume no collisions - } - - private static void createMap(InputStream in) throws Exception { - BufferedReader br = new BufferedReader(new InputStreamReader(in)); - String line = null; - while ((line = br.readLine()) != null) { - if (line.contains("tid=")) { - parseLine(line); - } - } - } - - // currently stores thread states to track locking periods - static class ThreadHistory { - public static Map tidHistoryMap - = new HashMap<>(); - private static final int HISTORY_SIZE = 60; // 60 * samplingInterval - public static void add(long tid, long value) { - CircularLongArray arr = tidHistoryMap.get(tid); - if (arr == null) { - arr = new CircularLongArray(HISTORY_SIZE); - arr.add(value); - tidHistoryMap.put(tid, arr); - } else { - arr.add(value); - } - } - public static void cleanup() { - long curTime = System.currentTimeMillis(); - for (Iterator> it = - tidHistoryMap.entrySet().iterator(); - it.hasNext();) { - Map.Entry me = it.next(); - CircularLongArray arr = me.getValue(); - // delete items updated older than 300s - if (curTime - arr.lastWriteTimestamp > HISTORY_SIZE * samplingInterval * 1.0e3) { - it.remove(); - } - } - } - } - - // models a fixed-capacity queue that is append-only - // not thread-safe - static class CircularLongArray { - ArrayList list = null; - public long lastWriteTimestamp; - private long totalValue; - private int startidx; - private int capacity; - CircularLongArray(int capacity) { - list = new ArrayList<>(capacity); - this.capacity = capacity; - totalValue = 0; - startidx = 0; - lastWriteTimestamp = 0; - } - public boolean add(long e) { - lastWriteTimestamp = System.currentTimeMillis(); - if (list.size() < capacity) { - // can only happen if startidx == 0 - if (startidx != 0) { - return false; - } else { - totalValue += e; - return list.add(e); - } - } - totalValue -= list.get(startidx); - totalValue += e; - list.set(startidx, e); - startidx = (startidx + 1) % capacity; - return true; - } - public double getAvgValue() { - return list.size() == 0 ? 0 : 1.0 * totalValue / list.size(); - } - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/AllMetrics.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/AllMetrics.java deleted file mode 100644 index 876263b4..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/AllMetrics.java +++ /dev/null @@ -1,909 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics; - -/** - * Contract between reader and writer. Writer write using the same values of - * these enums as json keys (See all MetricStatus's subclasses in - * com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors), while reader creates db tables using these - * keys as column names and extract values using these keys. You should - * make sure the the field names in the MetricStatus's subclasses and enum - * names match. Also, when you change anything, modify JsonKeyTest accordingly. - * We use camelCase instead of the usual capital case for enum members because - * they have better readability for the above use cases. - * - */ -public class AllMetrics { - // metric name (not complete, only metrics use the json format and contains - // numeric values. Will add more when needed) - public enum MetricName { - CIRCUIT_BREAKER, HEAP_METRICS, DISK_METRICS, TCP_METRICS, IP_METRICS, - THREAD_POOL, SHARD_STATS, MASTER_PENDING - } - - // we don't store node details as a metric on reader side database. We - // use the information as part of http response. - public enum NodeDetailColumns { - ID(Constants.ID_VALUE), - HOST_ADDRESS(Constants.HOST_ADDRESS_VALUE); - - private final String value; - - NodeDetailColumns(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String ID_VALUE = "ID"; - public static final String HOST_ADDRESS_VALUE = "HOST_ADDRESS"; - } - } - - // contents of metrics - public enum GCType { - TOT_YOUNG_GC(Constants.TOT_YOUNG_GC_VALUE), - TOT_FULL_GC(Constants.TOT_FULL_GC_VALUE), - SURVIVOR(Constants.SURVIVOR_VALUE), - PERM_GEN(Constants.PERM_GEN_VALUE), - OLD_GEN(Constants.OLD_GEN_VALUE), - EDEN(Constants.EDEN_VALUE), - NON_HEAP(Constants.NON_HEAP_VALUE), - HEAP(Constants.HEAP_VALUE); - - private final String value; - - GCType(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String TOT_YOUNG_GC_VALUE = "totYoungGC"; - public static final String TOT_FULL_GC_VALUE = "totFullGC"; - public static final String SURVIVOR_VALUE = "Survivor"; - public static final String PERM_GEN_VALUE = "PermGen"; - public static final String OLD_GEN_VALUE = "OldGen"; - public static final String EDEN_VALUE = "Eden"; - public static final String NON_HEAP_VALUE = "NonHeap"; - public static final String HEAP_VALUE = "Heap"; - } - } - - // column names of database table - public enum CircuitBreakerDimension implements MetricDimension { - CB_TYPE(Constants.TYPE_VALUE); - - private final String value; - - CircuitBreakerDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String TYPE_VALUE = "CBType"; - } - } - - // cannot use limit as it is a keyword in sql - public enum CircuitBreakerValue implements MetricValue { - CB_ESTIMATED_SIZE(Constants.ESTIMATED_VALUE), - CB_TRIPPED_EVENTS(Constants.TRIPPED_VALUE), - CB_CONFIGURED_SIZE(Constants.LIMIT_CONFIGURED_VALUE); - - private final String value; - - CircuitBreakerValue(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String ESTIMATED_VALUE = "CB_EstimatedSize"; - - public static final String TRIPPED_VALUE = "CB_TrippedEvents"; - - public static final String LIMIT_CONFIGURED_VALUE = - "CB_ConfiguredSize"; - } - } - - public enum HeapDimension implements MetricDimension { - MEM_TYPE(Constants.TYPE_VALUE); - - private final String value; - - HeapDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String TYPE_VALUE = "MemType"; - } - } - - public enum HeapValue implements MetricValue { - GC_COLLECTION_EVENT(Constants.COLLECTION_COUNT_VALUE), - GC_COLLECTION_TIME(Constants.COLLECTION_TIME_VALUE), - HEAP_COMMITTED(Constants.COMMITTED_VALUE), - HEAP_INIT(Constants.INIT_VALUE), - HEAP_MAX(Constants.MAX_VALUE), - HEAP_USED(Constants.USED_VALUE); - - private final String value; - - HeapValue(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String COLLECTION_COUNT_VALUE = "GC_Collection_Event"; - - public static final String COLLECTION_TIME_VALUE = "GC_Collection_Time"; - - public static final String COMMITTED_VALUE = "Heap_Committed"; - - public static final String INIT_VALUE = "Heap_Init"; - - public static final String MAX_VALUE = "Heap_Max"; - - public static final String USED_VALUE = "Heap_Used"; - } - } - - public enum DiskDimension implements MetricDimension { - DISK_NAME(Constants.NAME_VALUE); - - private final String value; - - DiskDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String NAME_VALUE = "DiskName"; - } - } - - public enum DiskValue implements MetricValue { - DISK_UTILIZATION(Constants.UTIL_VALUE), - DISK_WAITTIME(Constants.WAIT_VALUE), - DISK_SERVICE_RATE(Constants.SRATE_VALUE); - - private final String value; - - DiskValue(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String UTIL_VALUE = "Disk_Utilization"; - - public static final String WAIT_VALUE = "Disk_WaitTime"; - - public static final String SRATE_VALUE = "Disk_ServiceRate"; - } - } - - public enum TCPDimension implements MetricDimension { - DEST_ADDR(Constants.DEST_VALUE); - - private final String value; - - TCPDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String DEST_VALUE = "DestAddr"; - } - } - - public enum TCPValue implements MetricValue { - Net_TCP_NUM_FLOWS(Constants.NUM_FLOWS_VALUE), - Net_TCP_TXQ(Constants.TXQ_VALUE), - Net_TCP_RXQ(Constants.RXQ_VALUE), - Net_TCP_LOST(Constants.CUR_LOST_VALUE), - Net_TCP_SEND_CWND(Constants.SEND_CWND_VALUE), - Net_TCP_SSTHRESH(Constants.SSTHRESH_VALUE); - - private final String value; - - TCPValue(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String NUM_FLOWS_VALUE = "Net_TCP_NumFlows"; - - public static final String TXQ_VALUE = "Net_TCP_TxQ"; - - public static final String RXQ_VALUE = "Net_TCP_RxQ"; - - public static final String CUR_LOST_VALUE = "Net_TCP_Lost"; - - public static final String SEND_CWND_VALUE = "Net_TCP_SendCWND"; - - public static final String SSTHRESH_VALUE = "Net_TCP_SSThresh"; - } - } - - public enum IPDimension implements MetricDimension { - DIRECTION(Constants.DIRECTION_VALUE); - - private final String value; - - IPDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String DIRECTION_VALUE = "Direction"; - } - } - - public enum IPValue implements MetricValue { - NET_PACKET_RATE4(Constants.PACKET_RATE4_VALUE), - NET_PACKET_DROP_RATE4(Constants.DROP_RATE4_VALUE), - NET_PACKET_RATE6(Constants.PACKET_RATE6_VALUE), - NET_PACKET_DROP_RATE6(Constants.DROP_RATE6_VALUE), - NET_THROUGHPUT(Constants.THROUGHPUT_VALUE); - - private final String value; - - IPValue(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String PACKET_RATE4_VALUE = "Net_PacketRate4"; - public static final String DROP_RATE4_VALUE = "Net_PacketDropRate4"; - public static final String PACKET_RATE6_VALUE = "Net_PacketRate6"; - public static final String DROP_RATE6_VALUE = "Net_PacketDropRate6"; - public static final String THROUGHPUT_VALUE = "Net_Throughput"; - } - } - - public enum ThreadPoolDimension implements MetricDimension { - THREAD_POOL_TYPE(Constants.TYPE_VALUE); - - private final String value; - - ThreadPoolDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String TYPE_VALUE = "ThreadPoolType"; - } - } - - public enum ThreadPoolValue implements MetricValue { - THREADPOOL_QUEUE_SIZE(Constants.QUEUE_SIZE_VALUE), - THREADPOOL_REJECTED_REQS(Constants.REJECTED_VALUE), - THREADPOOL_TOTAL_THREADS(Constants.THREADS_COUNT_VALUE), - THREADPOOL_ACTIVE_THREADS(Constants.THREADS_ACTIVE_VALUE); - - private final String value; - - ThreadPoolValue(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String QUEUE_SIZE_VALUE = "ThreadPool_QueueSize"; - public static final String REJECTED_VALUE = "ThreadPool_RejectedReqs"; - public static final String THREADS_COUNT_VALUE = "ThreadPool_TotalThreads"; - public static final String THREADS_ACTIVE_VALUE = "ThreadPool_ActiveThreads"; - } - } - - // extra dimension values come from other places (e.g., file path) instead - // of metric files themselves - public enum ShardStatsDerivedDimension implements MetricDimension { - INDEX_NAME(Constants.INDEX_NAME_VALUE), - SHARD_ID(Constants.SHARD_ID_VALUE); - - private final String value; - - ShardStatsDerivedDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String INDEX_NAME_VALUE = - CommonDimension.INDEX_NAME.toString(); - - public static final String SHARD_ID_VALUE = - CommonDimension.SHARD_ID.toString(); - } - } - - public enum ShardStatsValue implements MetricValue { - INDEXING_THROTTLE_TIME(Constants.INDEXING_THROTTLE_TIME_VALUE), - CACHE_QUERY_HIT(Constants.QUEY_CACHE_HIT_COUNT_VALUE), - CACHE_QUERY_MISS(Constants.QUERY_CACHE_MISS_COUNT_VALUE), - CACHE_QUERY_SIZE(Constants.QUERY_CACHE_IN_BYTES_VALUE), - CACHE_FIELDDATA_EVICTION(Constants.FIELDDATA_EVICTION_VALUE), - CACHE_FIELDDATA_SIZE(Constants.FIELD_DATA_IN_BYTES_VALUE), - CACHE_REQUEST_HIT(Constants.REQUEST_CACHE_HIT_COUNT_VALUE), - CACHE_REQUEST_MISS(Constants.REQUEST_CACHE_MISS_COUNT_VALUE), - CACHE_REQUEST_EVICTION(Constants.REQUEST_CACHE_EVICTION_VALUE), - CACHE_REQUEST_SIZE(Constants.REQUEST_CACHE_IN_BYTES_VALUE), - REFRESH_EVENT(Constants.REFRESH_COUNT_VALUE), - REFRESH_TIME(Constants.REFRESH_TIME_VALUE), - FLUSH_EVENT(Constants.FLUSH_COUNT_VALUE), - FLUSH_TIME(Constants.FLUSH_TIME_VALUE), - MERGE_EVENT(Constants.MERGE_COUNT_VALUE), - MERGE_TIME(Constants.MERGE_TIME_VALUE), - MERGE_CURRENT_EVENT(Constants.MERGE_CURRENT_VALUE), - INDEXING_BUFFER(Constants.INDEX_BUFFER_BYTES_VALUE), - SEGMENTS_TOTAL(Constants.SEGMENTS_COUNT_VALUE), - SEGMENTS_MEMORY(Constants.SEGMENTS_MEMORY_VALUE), - TERMS_MEMORY(Constants.TERMS_MEMORY_VALUE), - STORED_FIELDS_MEMORY(Constants.STORED_FIELDS_MEMORY_VALUE), - TERM_VECTOR_MEMORY(Constants.TERM_VECTOR_MEMORY_VALUE), - NORMS_MEMORY(Constants.NORMS_MEMORY_VALUE), - POINTS_MEMORY(Constants.POINTS_MEMORY_VALUE), - DOC_VALUES_MEMORY(Constants.DOC_VALUES_MEMORY_VALUE), - INDEX_WRITER_MEMORY(Constants.INDEX_WRITER_MEMORY_VALUE), - VERSION_MAP_MEMORY(Constants.VERSION_MAP_MEMORY_VALUE), - BITSET_MEMORY(Constants.BITSET_MEMORY_VALUE); - - private final String value; - - ShardStatsValue(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String INDEXING_THROTTLE_TIME_VALUE = "Indexing_ThrottleTime"; - - public static final String QUEY_CACHE_HIT_COUNT_VALUE = "Cache_Query_Hit"; - - public static final String QUERY_CACHE_MISS_COUNT_VALUE = "Cache_Query_Miss"; - - public static final String QUERY_CACHE_IN_BYTES_VALUE = "Cache_Query_Size"; - - public static final String FIELDDATA_EVICTION_VALUE = "Cache_FieldData_Eviction"; - - public static final String FIELD_DATA_IN_BYTES_VALUE = "Cache_FieldData_Size"; - - public static final String REQUEST_CACHE_HIT_COUNT_VALUE = "Cache_Request_Hit"; - - public static final String REQUEST_CACHE_MISS_COUNT_VALUE = "Cache_Request_Miss"; - - public static final String REQUEST_CACHE_EVICTION_VALUE = "Cache_Request_Eviction"; - - public static final String REQUEST_CACHE_IN_BYTES_VALUE = "Cache_Request_Size"; - - public static final String REFRESH_COUNT_VALUE = "Refresh_Event"; - - public static final String REFRESH_TIME_VALUE = "Refresh_Time"; - - public static final String FLUSH_COUNT_VALUE = "Flush_Event"; - - public static final String FLUSH_TIME_VALUE = "Flush_Time"; - - public static final String MERGE_COUNT_VALUE = "Merge_Event"; - - public static final String MERGE_TIME_VALUE = "Merge_Time"; - - public static final String MERGE_CURRENT_VALUE = "Merge_CurrentEvent"; - - public static final String INDEX_BUFFER_BYTES_VALUE = "Indexing_Buffer"; - - public static final String SEGMENTS_COUNT_VALUE = "Segments_Total"; - - public static final String SEGMENTS_MEMORY_VALUE = "Segments_Memory"; - - public static final String TERMS_MEMORY_VALUE = "Terms_Memory"; - - public static final String STORED_FIELDS_MEMORY_VALUE = "StoredFields_Memory"; - - public static final String TERM_VECTOR_MEMORY_VALUE = "TermVectors_Memory"; - - public static final String NORMS_MEMORY_VALUE = "Norms_Memory"; - - public static final String POINTS_MEMORY_VALUE = "Points_Memory"; - - public static final String DOC_VALUES_MEMORY_VALUE = "DocValues_Memory"; - - public static final String INDEX_WRITER_MEMORY_VALUE = "IndexWriter_Memory"; - - public static final String VERSION_MAP_MEMORY_VALUE = "VersionMap_Memory"; - - public static final String BITSET_MEMORY_VALUE = "Bitset_Memory"; - } - } - - public enum MasterPendingValue implements MetricValue { - MASTER_PENDING_QUEUE_SIZE(Constants.PENDING_TASKS_COUNT_VALUE); - - private final String value; - - MasterPendingValue(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String PENDING_TASKS_COUNT_VALUE = "Master_PendingQueueSize"; - } - } - - - public enum OSMetrics { - CPU_UTILIZATION(Constants.CPU_VALUE), - PAGING_MAJ_FLT_RATE(Constants.PAGING_MAJFLT_VALUE), - PAGING_MIN_FLT_RATE(Constants.PAGING_MINFLT_VALUE), - PAGING_RSS(Constants.RSS_VALUE), - SCHED_RUNTIME(Constants.RUNTIME_VALUE), - SCHED_WAITTIME(Constants.WAITTIME_VALUE), - SCHED_CTX_RATE(Constants.CTXRATE_VALUE), - HEAP_ALLOC_RATE(Constants.HEAP_ALLOC_VALUE), - IO_READ_THROUGHPUT(Constants.READ_THROUGHPUT_VALUE), - IO_WRITE_THROUGHPUT(Constants.WRITE_THROUGHPUT_VALUE), - IO_TOT_THROUGHPUT(Constants.TOTAL_THROUGHPUT_VALUE), - IO_READ_SYSCALL_RATE(Constants.READ_SYSCALL_RATE_VALUE), - IO_WRITE_SYSCALL_RATE(Constants.WRITE_SYSCALL_RATE_VALUE), - IO_TOTAL_SYSCALL_RATE(Constants.TOTAL_SYSCALL_RATE_VALUE), - THREAD_BLOCKED_TIME(Constants.BLOCKED_TIME_VALUE), - THREAD_BLOCKED_EVENT(Constants.BLOCKED_COUNT_VALUE); - - private final String value; - - OSMetrics(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String CPU_VALUE = "CPU_Utilization"; - public static final String PAGING_MAJFLT_VALUE = "Paging_MajfltRate"; - public static final String PAGING_MINFLT_VALUE = "Paging_MinfltRate"; - public static final String RSS_VALUE = "Paging_RSS"; - public static final String RUNTIME_VALUE = "Sched_Runtime"; - public static final String WAITTIME_VALUE = "Sched_Waittime"; - public static final String CTXRATE_VALUE = "Sched_CtxRate"; - public static final String HEAP_ALLOC_VALUE = "Heap_AllocRate"; - public static final String READ_THROUGHPUT_VALUE = "IO_ReadThroughput"; - public static final String WRITE_THROUGHPUT_VALUE = "IO_WriteThroughput"; - public static final String TOTAL_THROUGHPUT_VALUE = "IO_TotThroughput"; - public static final String READ_SYSCALL_RATE_VALUE = "IO_ReadSyscallRate"; - public static final String WRITE_SYSCALL_RATE_VALUE = "IO_WriteSyscallRate"; - public static final String TOTAL_SYSCALL_RATE_VALUE = "IO_TotalSyscallRate"; - public static final String BLOCKED_TIME_VALUE = "Thread_Blocked_Time"; - public static final String BLOCKED_COUNT_VALUE = "Thread_Blocked_Event"; - } - } - - public enum MasterMetricDimensions implements MetricDimension { - MASTER_TASK_PRIORITY("MasterTaskPriority"), - MASTER_TASK_TYPE("MasterTaskType"), - MASTER_TASK_METADATA("MasterTaskMetadata"), - MASTER_TASK_QUEUE_TIME("MasterTaskQueueTime"), - MASTER_TASK_RUN_TIME("MasterTaskRunTime"), - MASTER_TASK_INSERT_ORDER("MasterTaskInsertOrder"); - - private final String value; - - MasterMetricDimensions(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - } - - public enum MasterMetricValues implements MetricValue { - //-todo : Migrate to CommonMetric.Constants - MASTER_TASK_QUEUE_TIME("Master_Task_Queue_Time"), - MASTER_TASK_RUN_TIME("Master_Task_Run_Time"), - START_TIME("StartTime"), - FINISH_TIME("FinishTime"); - - private final String value; - - MasterMetricValues(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - } - - public enum HttpDimension implements MetricDimension { - EXCEPTION(Constants.EXCEPTION_VALUE), - HTTP_RESP_CODE(Constants.HTTP_RESP_CODE_VALUE), - INDICES(Constants.INDICES_VALUE); - - private final String value; - - HttpDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String INDICES_VALUE = "Indices"; - public static final String EXCEPTION_VALUE = CommonDimension.EXCEPTION.toString(); - public static final String HTTP_RESP_CODE_VALUE = "HTTPRespCode"; - } - } - - public enum HttpMetric implements MetricValue { - START_TIME(Constants.START_TIME_VALUE), - HTTP_REQUEST_DOCS(Constants.HTTP_REQUEST_DOCS_VALUE), - FINISH_TIME(Constants.FINISH_TIME_VALUE), - HTTP_TOTAL_REQUESTS(Constants.HTTP_TOTAL_REQUESTS_VALUE); - - private final String value; - - HttpMetric(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String START_TIME_VALUE = CommonMetric.START_TIME.toString(); - public static final String FINISH_TIME_VALUE = CommonMetric.FINISH_TIME.toString(); - public static final String HTTP_REQUEST_DOCS_VALUE = "HTTP_RequestDocs"; - public static final String HTTP_TOTAL_REQUESTS_VALUE = "HTTP_TotalRequests"; - - } - } - - public enum ShardBulkDimension implements MetricDimension { - INDEX_NAME(Constants.INDEXNAME_VALUE), - SHARD_ID(Constants.SHARDID_VALUE), - PRIMARY(Constants.PRIMARY_VALUE), - EXCEPTION(Constants.EXCEPTION_VALUE), - FAILED(Constants.FAILED_VALUE); - - private final String value; - - ShardBulkDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String INDEXNAME_VALUE = CommonDimension.INDEX_NAME.toString(); - public static final String SHARDID_VALUE = CommonDimension.SHARD_ID.toString(); - public static final String PRIMARY_VALUE = "Primary"; - public static final String EXCEPTION_VALUE = CommonDimension.EXCEPTION.toString(); - public static final String FAILED_VALUE = CommonDimension.FAILED.toString(); - } - } - - public enum ShardBulkMetric implements MetricValue { - START_TIME(Constants.START_TIME_VALUE), - ITEM_COUNT(Constants.ITEM_COUNT_VALUE), - FINISH_TIME(Constants.FINISH_TIME_VALUE), - LATENCY(Constants.LATENCY_VALUE), - DOC_COUNT(Constants.DOC_COUNT); - - private final String value; - - ShardBulkMetric(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String START_TIME_VALUE = CommonMetric.START_TIME.toString(); - public static final String ITEM_COUNT_VALUE = "ItemCount"; - public static final String FINISH_TIME_VALUE = CommonMetric.FINISH_TIME.toString(); - public static final String LATENCY_VALUE = CommonMetric.LATENCY.toString(); - public static final String DOC_COUNT = "ShardBulkDocs"; - } - } - - public enum ShardOperationMetric implements MetricValue { - SHARD_OP_COUNT(Constants.SHARD_OP_COUNT_VALUE); - - private final String value; - - ShardOperationMetric(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String SHARD_OP_COUNT_VALUE = "ShardEvents"; - } - } - - public enum CommonDimension implements MetricDimension { - INDEX_NAME(Constants.INDEX_NAME_VALUE), - OPERATION(Constants.OPERATION_VALUE), - SHARD_ROLE(Constants.SHARD_ROLE_VALUE), - SHARD_ID(Constants.SHARDID_VALUE), - EXCEPTION(Constants.EXCEPTION_VALUE), - FAILED(Constants.FAILED_VALUE); - - private final String value; - - CommonDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String INDEX_NAME_VALUE = "IndexName"; - public static final String SHARDID_VALUE = "ShardID"; - public static final String OPERATION_VALUE = "Operation"; - public static final String SHARD_ROLE_VALUE = "ShardRole"; - public static final String EXCEPTION_VALUE = "Exception"; - public static final String FAILED_VALUE = "Failed"; - } - } - - public enum CommonMetric { - START_TIME(Constants.START_TIME_VALUE), - FINISH_TIME(Constants.FINISH_TIME_VALUE), - LATENCY(Constants.LATENCY_VALUE); - - private final String value; - - CommonMetric(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String START_TIME_VALUE = "StartTime"; - public static final String FINISH_TIME_VALUE = "FinishTime"; - public static final String LATENCY_VALUE = "Latency"; - } - } - - public enum EmptyDimension implements MetricDimension { - EMPTY(""); - - private final String value; - - EmptyDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - } - - public enum AggregatedOSDimension implements MetricDimension { - INDEX_NAME(CommonDimension.INDEX_NAME.toString()), - OPERATION(CommonDimension.OPERATION.toString()), - SHARD_ROLE(CommonDimension.SHARD_ROLE.toString()), - SHARD_ID(CommonDimension.SHARD_ID.toString()); - - private final String value; - - AggregatedOSDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - } - - public enum LatencyDimension implements MetricDimension { - OPERATION(CommonDimension.OPERATION.toString()), - EXCEPTION(CommonDimension.EXCEPTION.toString()), - INDICES(HttpDimension.INDICES.toString()), - HTTP_RESP_CODE(HttpDimension.HTTP_RESP_CODE.toString()), - SHARD_ID(CommonDimension.SHARD_ID.toString()), - INDEX_NAME(CommonDimension.INDEX_NAME.toString()), - SHARD_ROLE(CommonDimension.SHARD_ROLE.toString()); - - private final String value; - - LatencyDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - } - - public enum HttpOnlyDimension implements MetricDimension { - OPERATION(CommonDimension.OPERATION.toString()), - EXCEPTION(CommonDimension.EXCEPTION.toString()), - INDICES(HttpDimension.INDICES.toString()), - HTTP_RESP_CODE(HttpDimension.HTTP_RESP_CODE.toString()); - - private final String value; - - HttpOnlyDimension(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - } - - public enum MetricUnits { - CORES(Constants.CORES_VALUE), - COUNT_PER_SEC(Constants.COUNT_PER_SEC_VALUE), - COUNT(Constants.COUNT_VALUE), - PAGES(Constants.PAGES_VALUE), - SEC_PER_CONTEXT_SWITCH(Constants.SEC_PER_CONTEXT_SWITCH_VALUE), - BYTE_PER_SEC(Constants.BYTE_PER_SEC_VALUE), - SEC_PER_EVENT(Constants.SEC_PER_EVENT_VALUE), - MILLISECOND(Constants.MILLISECOND_VALUE), - BYTE(Constants.BYTE_VALUE), - PERCENT(Constants.PERCENT_VALUE), - MEGABYTE_PER_SEC(Constants.MEGABYTE_PER_SEC_VALUE), - SEGMENT_PER_FLOW(Constants.SEGMENT_PER_FLOW_VALUE), - BYTE_PER_FLOW(Constants.BYTE_PER_FLOW_VALUE), - PACKET_PER_SEC(Constants.PACKET_PER_SEC_VALUE); - - private final String value; - - MetricUnits(String value) { - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public static class Constants { - public static final String CORES_VALUE = "cores"; - public static final String COUNT_PER_SEC_VALUE = "count/s"; - public static final String COUNT_VALUE = "count"; - public static final String PAGES_VALUE = "pages"; - public static final String SEC_PER_CONTEXT_SWITCH_VALUE = "s/ctxswitch"; - public static final String BYTE_PER_SEC_VALUE = "B/s"; - public static final String SEC_PER_EVENT_VALUE = "s/event"; - public static final String MILLISECOND_VALUE = "ms"; - public static final String BYTE_VALUE = "B"; - public static final String PERCENT_VALUE = "%"; - public static final String MEGABYTE_PER_SEC_VALUE = "MB/s"; - public static final String SEGMENT_PER_FLOW_VALUE = "segments/flow"; - public static final String BYTE_PER_FLOW_VALUE = "B/flow"; - public static final String PACKET_PER_SEC_VALUE = "packets/s"; - - } - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/MetricDimension.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/MetricDimension.java deleted file mode 100644 index 19aaeb47..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/MetricDimension.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics; - -/** - * This helps writing a general parser. Given a MetricDimension, I can parse the - * metric file using the values provided by the MetricDimension enum. I don't - * need to hardcode the exact enum name in the parser. The parser only needs to - * know this enum has a metric's dimensions and use its members as Json key to - * parse out the concrete metric dimensions. See - * src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricProperties.java - * - * - */ -public interface MetricDimension { - -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/MetricValue.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/MetricValue.java deleted file mode 100644 index 151dcd2c..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/MetricValue.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics; - -/** - * This helps writing a general parser. Given a MetricValue, I can parse the - * metric file using the values provided by the MetricValue enum. I don't need - * to hardcode the exact enum name in the parser. The parser only needs to know - * this enum has a metric's values and use its members as Json key to parse out - * the concrete metric values. See - * src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricProperties.java - * - * - */ -public interface MetricValue { - -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/MetricsConfiguration.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/MetricsConfiguration.java deleted file mode 100644 index 1116b8bf..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/MetricsConfiguration.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics; - -import java.util.HashMap; -import java.util.Map; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.CircuitBreakerCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.DisksCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.HeapMetricsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceEventMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MetricsPurgeActivity; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NetworkE2ECollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NetworkInterfaceCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NodeDetailsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NodeStatsMetricsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ThreadPoolMetricsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm.GCMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm.HeapMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm.ThreadList; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.OSGlobals; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.ThreadCPU; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.ThreadDiskIO; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.ThreadSched; - -public class MetricsConfiguration { - public static final int SAMPLING_INTERVAL = 5000; - public static final int ROTATION_INTERVAL = 30000; - public static final int STATS_ROTATION_INTERVAL = 60000; - public static final int DELETION_INTERVAL = PluginSettings.instance().getMetricsDeletionInterval(); - - public static class MetricConfig { - public int samplingInterval; - public int rotationInterval; - public int deletionInterval; - - MetricConfig(int samplingInterval, - int rotationInterval, - int deletionInterval) { - this.samplingInterval = samplingInterval; - this.rotationInterval = rotationInterval; - this.deletionInterval = deletionInterval; - } - } - - public static final Map CONFIG_MAP = new HashMap<>(); - - static { - MetricConfig cdefault = new MetricConfig(SAMPLING_INTERVAL, 0, 0); - - CONFIG_MAP.put(ThreadCPU.class, cdefault); - CONFIG_MAP.put(ThreadDiskIO.class, cdefault); - CONFIG_MAP.put(ThreadSched.class, cdefault); - CONFIG_MAP.put(ThreadList.class, cdefault); - CONFIG_MAP.put(GCMetrics.class, cdefault); - CONFIG_MAP.put(HeapMetrics.class, cdefault); - CONFIG_MAP.put(NetworkE2ECollector.class, cdefault); - CONFIG_MAP.put(NetworkInterfaceCollector.class, cdefault); - CONFIG_MAP.put(OSGlobals.class, cdefault); - CONFIG_MAP.put(PerformanceAnalyzerMetrics.class, new MetricConfig(0, ROTATION_INTERVAL, 0)); - CONFIG_MAP.put(MetricsPurgeActivity.class, new MetricConfig(ROTATION_INTERVAL, 0, DELETION_INTERVAL)); - CONFIG_MAP.put(StatsCollector.class, new MetricConfig(STATS_ROTATION_INTERVAL, 0, 0)); - CONFIG_MAP.put(MasterServiceEventMetrics.class, new MetricConfig(1000, 0, 0)); - CONFIG_MAP.put(MasterServiceMetrics.class, cdefault); - CONFIG_MAP.put(DisksCollector.class, cdefault); - CONFIG_MAP.put(CircuitBreakerCollector.class, cdefault); - CONFIG_MAP.put(HeapMetricsCollector.class, cdefault); - CONFIG_MAP.put(NodeDetailsCollector.class, cdefault); - CONFIG_MAP.put(NodeStatsMetricsCollector.class, cdefault); - CONFIG_MAP.put(ThreadPoolMetricsCollector.class, cdefault); - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/MetricsProcessor.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/MetricsProcessor.java deleted file mode 100644 index b4d9b462..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/MetricsProcessor.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerPlugin; - -public interface MetricsProcessor { - - default String getMetricValues(long startTime, String... keysPath) { - return PerformanceAnalyzerMetrics.getMetric(getMetricsPath(startTime, keysPath)); - } - - default void saveMetricValues(String value, long startTime, String... keysPath) { - PerformanceAnalyzerPlugin.invokePrivileged(() -> PerformanceAnalyzerMetrics.emitMetric(getMetricsPath(startTime, keysPath), value)); - } - - default String getMetricValue(String metricName, long startTime, String... keys) { - return PerformanceAnalyzerMetrics.extractMetricValue(getMetricValues(startTime, keys), metricName); - } - - String getMetricsPath(long startTime, String... keysPath); -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/PerformanceAnalyzerMetrics.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/PerformanceAnalyzerMetrics.java deleted file mode 100644 index adbba4af..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/PerformanceAnalyzerMetrics.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.logging.log4j.util.Supplier; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; - -@SuppressWarnings("checkstyle:constantname") -public class PerformanceAnalyzerMetrics { - private static final Logger LOG = LogManager.getLogger(PerformanceAnalyzerMetrics.class); - public static final String sDevShmLocation = PluginSettings.instance().getMetricsLocation(); - public static final String sDevShmScratchLocation = "performanceanalyzer_scratch"; - public static final String sIndicesPath = "indices"; - public static final String sThreadPoolPath = "thread_pool"; - public static final String sThreadsPath = "threads"; - public static final String sCircuitBreakerPath = "circuit_breaker"; - public static final String sShardBulkPath = "shardbulk"; - public static final String sShardFetchPath = "shardfetch"; - public static final String sShardQueryPath = "shardquery"; - public static final String sMasterTaskPath = "master_task"; - public static final String sHttpPath = "http"; - public static final String sOSPath = "os_metrics"; - public static final String sHeapPath = "heap_metrics"; - public static final String sNodesPath = "node_metrics"; - public static final String sPendingTasksPath = "pending_tasks"; - public static final String sDisksPath = "disk_metrics"; - public static final String sTCPPath = "tcp_metrics"; - public static final String sIPPath = "ip_metrics"; - public static final String sKeyValueDelimitor = ":"; - public static final String sMetricNewLineDelimitor = System.getProperty("line.separator"); - public static final String START_FILE_NAME = "start"; - public static final String FINISH_FILE_NAME = "finish"; - public static final String MASTER_CURRENT = "current"; - public static final String MASTER_META_DATA = "metadata"; - public static final String METRIC_CURRENT_TIME = "current_time"; - - private static final int NUM_RETRIES_FOR_TMP_FILE = 10; - - private static final boolean IS_METRICS_LOG_ENABLED = - System.getProperty("performanceanalyzer.metrics.log.enabled", "False").equalsIgnoreCase("True"); - - private static final int sTimeInterval = - MetricsConfiguration.CONFIG_MAP.get(PerformanceAnalyzerMetrics.class).rotationInterval; - - public static long getTimeInterval(long startTime) { - return getTimeInterval(startTime, sTimeInterval); - } - - public static long getTimeInterval(long startTime, int timeInterval) { - return (startTime / timeInterval) * timeInterval; - } - - public static String getCurrentTimeMetric() { - return METRIC_CURRENT_TIME + sKeyValueDelimitor + System.currentTimeMillis(); - } - - public static String generatePath(long startTime, String... keysPath) { - Path sDevShmLocationPath = Paths.get(sDevShmLocation) - .resolve(Paths.get(String.valueOf(PerformanceAnalyzerMetrics.getTimeInterval(startTime)), keysPath)); - return sDevShmLocationPath.toString(); - } - - public static void addMetricEntry(StringBuilder value, String metricKey, String metricValue) { - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor).append(metricKey) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor).append(metricValue); - } - - public static void addMetricEntry(StringBuilder value, String metricKey, long metricValue) { - value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor).append(metricKey) - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor).append(metricValue); - } - - public static void emitMetric(String keyPath, String value) { - File file = new File(keyPath); - if (IS_METRICS_LOG_ENABLED) { - LOG.info(keyPath + "\n" + value); - } - - try { - java.nio.file.Files.createDirectories(file.getParentFile().toPath()); - } catch (IOException ex) { - LOG.debug( - (Supplier) () -> new ParameterizedMessage( - "Error In Creating Directories: {} for keyPath:{} with ExceptionCode: {}", - ex.toString(), keyPath, StatExceptionCode.METRICS_WRITE_ERROR.toString()), - ex); - StatsCollector.instance().logException(StatExceptionCode.METRICS_WRITE_ERROR); - return; - } - - File tmpFile = null; - try { - tmpFile = writeToTmp(keyPath, value); - } catch (Exception ex) { - LOG.debug( - (Supplier) () -> new ParameterizedMessage( - "Error in Writing to Tmp File: {} for keyPath:{} with ExceptionCode: {}", - ex.toString(), keyPath, StatExceptionCode.METRICS_WRITE_ERROR.toString()), - ex); - StatsCollector.instance().logException(StatExceptionCode.METRICS_WRITE_ERROR); - return; - } - - try { - tmpFile.renameTo(file); - } catch (Exception ex) { - LOG.debug( - (Supplier) () -> new ParameterizedMessage( - "Error in Renaming Tmp File: {} for keyPath:{} with ExceptionCode: {}", - ex.toString(), keyPath, StatExceptionCode.METRICS_WRITE_ERROR.toString()), - ex); - StatsCollector.instance().logException(StatExceptionCode.METRICS_WRITE_ERROR); - } - } - - private static File writeToTmp(String keyPath, String value) throws Exception { - int numRetries = 0; - - //- try 10 times to avoid the hash code collision - while (numRetries < NUM_RETRIES_FOR_TMP_FILE) { - //- creating a tmp file under: /dev/shm/performanceanalyzer//performanceanalyzer_scrtach/ - //- In case rename fails, we don't need to delete this, auto purge will happen when the TIMESTAMP bucket will purged - //- To avoid collisions, temp file name chosen as: - //- hashcode of (absolue metric file path + value + current time nano seconds) - StringBuilder tmp = new StringBuilder().append(keyPath).append(value).append(String.valueOf(System.nanoTime())); - File file = new File(PerformanceAnalyzerMetrics.generatePath(System.currentTimeMillis(), sDevShmScratchLocation, - String.valueOf(tmp.toString().hashCode()))); - java.nio.file.Files.createDirectories(file.getParentFile().toPath()); - if (file.createNewFile()) { - try (FileOutputStream fos = new FileOutputStream(file);) { - fos.write(value.getBytes()); - } - return file; - } - numRetries++; - } - throw new Exception("Tmp file not able to create after " + NUM_RETRIES_FOR_TMP_FILE + " retries"); - } - - public static String getMetric(long startTime, String... keysPath) { - return getMetric(generatePath(startTime, keysPath)); - } - - public static String getMetric(String keyPath) { - try { - return new String(Files.readAllBytes(Paths.get(keyPath))); - } catch (Exception ex) { - //-todo logging -// ex.printStackTrace(); - return ""; - } - } - - public static String extractMetricValue(String metricVal, String key) { - int startIndex = metricVal.indexOf(key); - - if (startIndex != -1) { - startIndex = metricVal.indexOf(sKeyValueDelimitor, startIndex); - int endIndex = metricVal.indexOf(sMetricNewLineDelimitor, startIndex + 1); - - if (endIndex == -1) { - endIndex = metricVal.length(); - } - return metricVal.substring(startIndex + 1, endIndex); - } - return null; - } - - public static void removeMetrics(String keyPath) { - removeMetrics(new File(keyPath)); - } - - public static void removeMetrics(File keyPathFile) { - if (keyPathFile.isDirectory()) { - String[] children = keyPathFile.list(); - for (int i = 0; i < children.length; i++) { - removeMetrics(new File(keyPathFile, children[i])); - } - } - try { - keyPathFile.delete(); - } catch (Exception ex) { - StatsCollector.instance().logException(StatExceptionCode.METRICS_REMOVE_ERROR); - LOG.debug( - (Supplier) () -> new ParameterizedMessage( - "Error in deleting file: {} for keyPath:{} with ExceptionCode: {}", - ex.toString(), keyPathFile.getAbsolutePath(), StatExceptionCode.METRICS_REMOVE_ERROR.toString()), - ex); - } - } - - public static String getJsonCurrentMilliSeconds() { - return new StringBuilder().append("{\"") - .append(PerformanceAnalyzerMetrics.METRIC_CURRENT_TIME).append("\"") - .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) - .append(System.currentTimeMillis()).append("}").toString(); - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/ThreadIDUtil.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/ThreadIDUtil.java deleted file mode 100644 index 63c5cfa6..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/ThreadIDUtil.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm.ThreadList; - -public final class ThreadIDUtil { - private ThreadIDUtil() { } - - public static final ThreadIDUtil INSTANCE = new ThreadIDUtil(); - - public long getNativeCurrentThreadId() { - - return getNativeThreadId(Thread.currentThread().getId()); - } - - public long getNativeThreadId(long jTid) { - ThreadList.ThreadState threadState1 = ThreadList.getThreadState(jTid); - - long nid = -1; - if (threadState1 != null) { - nid = threadState1.nativeTid; - } - - return nid; - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/CPUPagingActivityGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/CPUPagingActivityGenerator.java deleted file mode 100644 index 768d9daa..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/CPUPagingActivityGenerator.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator; - -public interface CPUPagingActivityGenerator { - - // This method will be called before all following get methods - // to make sure that all information exists for a thread id - boolean hasPagingActivity(String threadId); - - double getCPUUtilization(String threadId); - double getMajorFault(String threadId); - double getMinorFault(String threadId); - double getResidentSetSize(String threadId); - void addSample(); -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/DiskIOMetricsGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/DiskIOMetricsGenerator.java deleted file mode 100644 index 3671ad1c..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/DiskIOMetricsGenerator.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator; - -public interface DiskIOMetricsGenerator { - - // This method will be called before all following get methods - // to make sure that all information exists for a thread id - boolean hasDiskIOMetrics(String threadId); - - // these metrics include page cache activity; - // only explicit syscalls: NO mmaps (majflts include mmaps) - double getAvgReadThroughputBps(String threadId); - double getAvgWriteThroughputBps(String threadId); - double getAvgTotalThroughputBps(String threadId); - double getAvgReadSyscallRate(String threadId); - double getAvgWriteSyscallRate(String threadId); - double getAvgTotalSyscallRate(String threadId); - void addSample(); -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/DiskMetricsGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/DiskMetricsGenerator.java deleted file mode 100644 index 71ead045..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/DiskMetricsGenerator.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator; - -import java.util.Set; - -public interface DiskMetricsGenerator { - Set getAllDisks(); - double getDiskUtilization(String disk); - double getAwait(String disk); - double getServiceRate(String disk); - void addSample(); -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/IPMetricsGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/IPMetricsGenerator.java deleted file mode 100644 index 3c64f928..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/IPMetricsGenerator.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator; - -public interface IPMetricsGenerator { - double getInPacketRate4(); - double getOutPacketRate4(); - double getInDropRate4(); - double getOutDropRate4(); - double getInPacketRate6(); - double getOutPacketRate6(); - double getInDropRate6(); - double getOutDropRate6(); - double getInBps(); - double getOutBps(); - void addSample(); -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/OSMetricsGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/OSMetricsGenerator.java deleted file mode 100644 index db55423f..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/OSMetricsGenerator.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator; - -import java.util.Set; - -public interface OSMetricsGenerator { - - String getPid(); - CPUPagingActivityGenerator getPagingActivityGenerator(); - SchedMetricsGenerator getSchedMetricsGenerator(); - Set getAllThreadIds(); - DiskIOMetricsGenerator getDiskIOMetricsGenerator(); - TCPMetricsGenerator getTCPMetricsGenerator(); - IPMetricsGenerator getIPMetricsGenerator(); - DiskMetricsGenerator getDiskMetricsGenerator(); - -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/SchedMetricsGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/SchedMetricsGenerator.java deleted file mode 100644 index 2d954333..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/SchedMetricsGenerator.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator; - -public interface SchedMetricsGenerator { - - // This method will be called before all following get methods - // to make sure that all information exists for a thread id - boolean hasSchedMetrics(String threadId); - - double getAvgRuntime(String threadId); - double getAvgWaittime(String threadId); - double getContextSwitchRate(String threadId); - void addSample(); -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/TCPMetricsGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/TCPMetricsGenerator.java deleted file mode 100644 index 5c6e0c1b..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/TCPMetricsGenerator.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator; - -import java.util.Set; - -public interface TCPMetricsGenerator { - - Set getAllDestionationIps(); - int getNumberOfFlows(String ip); - double getTransmitQueueSize(String ip); - double getReceiveQueueSize(String ip); - double getCurrentLost(String ip); - double getSendCongestionWindow(String ip); - double getSlowStartThreshold(String ip); - void addSample(); -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxCPUPagingActivityGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxCPUPagingActivityGenerator.java deleted file mode 100644 index 8b6659b7..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxCPUPagingActivityGenerator.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux; - - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.CPUPagingActivityGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.ThreadCPU; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -public class LinuxCPUPagingActivityGenerator implements CPUPagingActivityGenerator { - - private Map cpu; - private Map pagingActivities; - public LinuxCPUPagingActivityGenerator() { - cpu = new HashMap<>(); - pagingActivities = new HashMap<>(); - } - - @Override - public double getCPUUtilization(final String threadId) { - - return cpu.getOrDefault(threadId, 0.0); - } - - @Override - public double getMajorFault(final String threadId) { - - return pagingActivities.get(threadId)[0]; - } - - @Override - public double getMinorFault(final String threadId) { - - return pagingActivities.get(threadId)[1]; - } - - @Override - public double getResidentSetSize(final String threadId) { - - return pagingActivities.get(threadId)[2]; - } - - @Override - public boolean hasPagingActivity(final String threadId) { - - return pagingActivities.containsKey(threadId); - } - - @Override - public void addSample() { - - cpu.clear(); - pagingActivities.clear(); - ThreadCPU.INSTANCE.addSample(); - } - - public void setCPUUtilization(final String threadId, final Double cpuUtilization) { - - cpu.put(threadId, cpuUtilization); - } - - public Set getAllThreadIds() { - - return cpu.keySet(); - } - - public void setPagingActivities(final String threadId, final Double[] activityes) { - pagingActivities.put(threadId, activityes); - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxDiskIOMetricsGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxDiskIOMetricsGenerator.java deleted file mode 100644 index 77c818ba..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxDiskIOMetricsGenerator.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.DiskIOMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.ThreadDiskIO; - -import java.util.HashMap; -import java.util.Map; - -public class LinuxDiskIOMetricsGenerator implements DiskIOMetricsGenerator { - - - private Map diskIOMetricsMap; - public LinuxDiskIOMetricsGenerator() { - diskIOMetricsMap = new HashMap<>(); - } - - @Override - public double getAvgReadThroughputBps(final String threadId) { - - return diskIOMetricsMap.get(threadId).avgReadThroughputBps; - } - - @Override - public double getAvgReadSyscallRate(final String threadId) { - - return diskIOMetricsMap.get(threadId).avgReadSyscallRate; - } - - @Override - public double getAvgWriteThroughputBps(final String threadId) { - - return diskIOMetricsMap.get(threadId).avgWriteThroughputBps; - } - - @Override - public double getAvgWriteSyscallRate(final String threadId) { - - return diskIOMetricsMap.get(threadId).avgWriteSyscallRate; - } - - @Override - public double getAvgTotalThroughputBps(final String threadId) { - - return diskIOMetricsMap.get(threadId).avgTotalThroughputBps; - } - - @Override - public double getAvgTotalSyscallRate(final String threadId) { - - return diskIOMetricsMap.get(threadId).avgTotalSyscallRate; - } - - @Override - public boolean hasDiskIOMetrics(final String threadId) { - - return diskIOMetricsMap.containsKey(threadId); - } - - @Override - public void addSample() { - ThreadDiskIO.addSample(); - } - - - public void setDiskIOMetrics(final String threadId, final ThreadDiskIO.IOMetrics ioMetrics) { - diskIOMetricsMap.put(threadId, ioMetrics); - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxDiskMetricsGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxDiskMetricsGenerator.java deleted file mode 100644 index 8ac82e7c..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxDiskMetricsGenerator.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.DiskMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.hwnet.Disks; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.DiskMetricsGenerator; - -import java.util.Map; -import java.util.Set; - -public class LinuxDiskMetricsGenerator implements DiskMetricsGenerator { - - private Map diskMetricsMap; - - @Override - public Set getAllDisks() { - return diskMetricsMap.keySet(); - } - - @Override - public double getDiskUtilization(final String disk) { - - return diskMetricsMap.get(disk).utilization; - } - - @Override - public double getAwait(final String disk) { - - return diskMetricsMap.get(disk).await; - } - - @Override - public double getServiceRate(final String disk) { - - return diskMetricsMap.get(disk).serviceRate; - } - - @Override - public void addSample() { - Disks.addSample(); - } - - public void setDiskMetricsMap(final Map map) { - - diskMetricsMap = map; - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxIPMetricsGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxIPMetricsGenerator.java deleted file mode 100644 index 52042d5f..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxIPMetricsGenerator.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NetInterfaceSummary; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.hwnet.NetworkInterface; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.IPMetricsGenerator; - -public class LinuxIPMetricsGenerator implements IPMetricsGenerator { - - - private NetInterfaceSummary inNetInterfaceSummary; - private NetInterfaceSummary outNetInterfaceSummary; - - @Override - public double getInPacketRate4() { - - return inNetInterfaceSummary.getPacketRate4(); - } - - @Override - public double getOutPacketRate4() { - - return outNetInterfaceSummary.getPacketRate4(); - } - - @Override - public double getInDropRate4() { - - return inNetInterfaceSummary.getDropRate4(); - } - - @Override - public double getOutDropRate4() { - - return outNetInterfaceSummary.getDropRate4(); - } - - @Override - public double getInPacketRate6() { - - return inNetInterfaceSummary.getPacketRate6(); - } - - @Override - public double getOutPacketRate6() { - - return outNetInterfaceSummary.getPacketRate6(); - } - - @Override - public double getInDropRate6() { - - return inNetInterfaceSummary.getDropRate6(); - } - - @Override - public double getOutDropRate6() { - - return outNetInterfaceSummary.getDropRate6(); - } - - @Override - public double getInBps() { - - return inNetInterfaceSummary.getBps(); - } - - @Override - public double getOutBps() { - - return outNetInterfaceSummary.getBps(); - } - - @Override - public void addSample() { - - NetworkInterface.addSample(); - } - - public void setInNetworkInterfaceSummary(final NetInterfaceSummary netInterfaceSummary) { - - this.inNetInterfaceSummary = netInterfaceSummary; - } - - public void setOutNetworkInterfaceSummary(final NetInterfaceSummary netInterfaceSummary) { - - this.outNetInterfaceSummary = netInterfaceSummary; - } - -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxOSMetricsGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxOSMetricsGenerator.java deleted file mode 100644 index 13938e37..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxOSMetricsGenerator.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.hwnet.Disks; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.hwnet.NetworkE2E; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.hwnet.NetworkInterface; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.DiskIOMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.DiskMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.IPMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.OSMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.CPUPagingActivityGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.SchedMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.TCPMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.OSGlobals; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.ThreadCPU; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.ThreadDiskIO; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.ThreadSched; - -import java.util.Set; - -public class LinuxOSMetricsGenerator implements OSMetricsGenerator { - - private static OSMetricsGenerator osMetricsGenerator; - static { - osMetricsGenerator = new LinuxOSMetricsGenerator(); - } - - public static OSMetricsGenerator getInstance() { - - return osMetricsGenerator; - } - - @Override - public String getPid() { - - return OSGlobals.getPid(); - } - - @Override - public CPUPagingActivityGenerator getPagingActivityGenerator() { - - return ThreadCPU.INSTANCE.getCPUPagingActivity(); - } - - @Override - public Set getAllThreadIds() { - return ThreadCPU.INSTANCE.getCPUPagingActivity().getAllThreadIds(); - } - - @Override - public DiskIOMetricsGenerator getDiskIOMetricsGenerator() { - - return ThreadDiskIO.getIOUtilization(); - } - - @Override - public SchedMetricsGenerator getSchedMetricsGenerator() { - - return ThreadSched.INSTANCE.getSchedLatency(); - } - - @Override - public TCPMetricsGenerator getTCPMetricsGenerator() { - - return NetworkE2E.getTCPMetricsHandler(); - } - - @Override - public IPMetricsGenerator getIPMetricsGenerator() { - - return NetworkInterface.getLinuxIPMetricsGenerator(); - } - - @Override - public DiskMetricsGenerator getDiskMetricsGenerator() { - - return Disks.getDiskMetricsHandler(); - } - -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxSchedMetricsGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxSchedMetricsGenerator.java deleted file mode 100644 index 5983e950..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxSchedMetricsGenerator.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.SchedMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.ThreadSched; - -import java.util.HashMap; -import java.util.Map; - -public class LinuxSchedMetricsGenerator implements SchedMetricsGenerator { - - private final Map schedMetricsMap; - - - public LinuxSchedMetricsGenerator() { - schedMetricsMap = new HashMap<>(); - } - - @Override - public double getAvgRuntime(final String threadId) { - - return schedMetricsMap.get(threadId).avgRuntime; - } - - @Override - public double getAvgWaittime(final String threadId) { - - return schedMetricsMap.get(threadId).avgWaittime; - } - - @Override - public double getContextSwitchRate(final String threadId) { - - return schedMetricsMap.get(threadId).contextSwitchRate; - } - - @Override - public boolean hasSchedMetrics(final String threadId) { - - return schedMetricsMap.containsKey(threadId); - } - - @Override - public void addSample() { - - schedMetricsMap.clear(); - ThreadSched.INSTANCE.addSample(); - } - - public void setSchedMetric(final String threadId, final ThreadSched.SchedMetrics schedMetrics) { - - schedMetricsMap.put(threadId, schedMetrics); - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxTCPMetricsGenerator.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxTCPMetricsGenerator.java deleted file mode 100644 index 7c601c16..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics_generator/linux/LinuxTCPMetricsGenerator.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.hwnet.NetworkE2E; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.TCPMetricsGenerator; - -import java.util.Map; -import java.util.Set; - -public class LinuxTCPMetricsGenerator implements TCPMetricsGenerator { - - private Map map; - - @Override - public Set getAllDestionationIps() { - return map.keySet(); - } - - @Override - public int getNumberOfFlows(final String ip) { - return (int)map.get(ip)[0]; - } - - @Override - public double getTransmitQueueSize(String ip) { - return map.get(ip)[1]; - } - - @Override - public double getReceiveQueueSize(String ip) { - return map.get(ip)[2]; - } - - @Override - public double getCurrentLost(String ip) { - return map.get(ip)[3]; - } - - @Override - public double getSendCongestionWindow(String ip) { - return map.get(ip)[4]; - } - - @Override - public double getSlowStartThreshold(String ip) { - return map.get(ip)[5]; - } - - @Override - public void addSample() { - NetworkE2E.addSample(); - } - - public void setTCPMetrics(final Map metrics) { - map = metrics; - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metricsdb/Dimensions.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metricsdb/Dimensions.java deleted file mode 100644 index a7dd382d..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metricsdb/Dimensions.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb; - -import java.util.Set; -import java.util.Map; -import java.util.HashMap; - -import org.jooq.impl.DSL; -import org.jooq.Field; - -public class Dimensions { - //Dimension is a key, value - private Map dimensions; - public Dimensions() { - this.dimensions = new HashMap<>(); - } - - public void put(String key, String value) { - this.dimensions.put(key, value); - } - - public String get(String key) { - return this.dimensions.get(key); - } - - public Map, String> getFieldMap() { - Map, String> fieldMap = new HashMap, String>(); - for (Map.Entry entry: dimensions.entrySet()) { - fieldMap.put(DSL.field(DSL.name(entry.getKey()), String.class), entry.getValue()); - } - return fieldMap; - } - - public Set getDimensionNames() { - return this.dimensions.keySet(); - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metricsdb/Metric.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metricsdb/Metric.java deleted file mode 100644 index 62f2abd1..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metricsdb/Metric.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb; - -public class Metric { - private String name; - private T sum; - private T avg; - private T min; - private T max; - - public Metric(String name, T value) { - this.name = name; - this.sum = value; - this.avg = value; - this.min = value; - this.max = value; - } - - public Metric(String name, T sum, T avg, T min, T max) { - this.name = name; - this.sum = sum; - this.avg = avg; - this.min = min; - this.max = max; - } - - public String getName() { - return this.name; - } - - public T getSum() { - return this.sum; - } - - public T getAvg() { - return this.avg; - } - - public T getMin() { - return this.min; - } - - public T getMax() { - return this.max; - } - - public Class getValueType() { - return this.sum.getClass(); - } - - //Unit test helper methods - public static Metric cpu(Double val) { - return new Metric("cpu", val); - } - - //Unit test helper methods - public static Metric rss(Double val) { - return new Metric("rss", val); - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metricsdb/MetricsDB.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metricsdb/MetricsDB.java deleted file mode 100644 index c7781493..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metricsdb/MetricsDB.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb; - -import java.io.File; -import java.sql.Connection; -import java.sql.DriverManager; -import java.util.ArrayList; -import java.util.List; -import java.util.HashSet; -import java.util.Arrays; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jooq.BatchBindStep; -import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.SQLDialect; -import org.jooq.Select; -import org.jooq.TableLike; -import org.jooq.impl.DSL; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.DBUtils; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader.Removable; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; - -/** - * On-disk database that holds a 5 second snapshot of all metrics. - * We create one table per metric. Every row contains four aggregations and any other relevant dimensions. - * - * Eg: - * CPU table - * |sum|avg|max|min| index|shard|role| - * +---+---+---+---+--------+-----+----+ - * | 5|2.5| 3| 2|sonested| 1| N/A| - * - * RSS table - * |sum|avg|max|min| index|shard|role| - * +---+---+---+---+---------+-----+----+ - * | 30| 15| 20| 10|nyc_taxis| 1| N/A| - */ -@SuppressWarnings("serial") -public class MetricsDB implements Removable { - - private static final Logger LOG = LogManager.getLogger(MetricsDB.class); - - private static final String DB_FILE_PREFIX_PATH_DEFAULT = "/tmp/metricsdb_"; - private static final String DB_FILE_PREFIX_PATH_CONF_NAME = "metrics-db-file-prefix-path"; - private static final String DB_URL = "jdbc:sqlite:"; - private final Connection conn; - private final DSLContext create; - public static final String SUM = "sum"; - public static final String AVG = "avg"; - public static final String MIN = "min"; - public static final String MAX = "max"; - public static final HashSet AGG_VALUES = new HashSet<>(Arrays.asList(SUM, AVG, MIN, MAX)); - private long windowStartTime; - - public String getDBFilePath() { - return PluginSettings.instance().getSettingValue(DB_FILE_PREFIX_PATH_CONF_NAME, DB_FILE_PREFIX_PATH_DEFAULT) - + Long.toString(windowStartTime); - } - - public MetricsDB(long windowStartTime) throws Exception { - this.windowStartTime = windowStartTime; - String url = DB_URL + getDBFilePath(); - conn = DriverManager.getConnection(url); - conn.setAutoCommit(false); - create = DSL.using(conn, SQLDialect.SQLITE); - } - - public void close() throws Exception { - conn.close(); - } - - public void createMetric(Metric metric, List dimensions) { - if (DBUtils.checkIfTableExists(create, metric.getName())) { - return; - } - - List> fields = DBUtils.getFieldsFromList(dimensions); - fields.add(DSL.field(SUM, metric.getValueType())); - fields.add(DSL.field(AVG, metric.getValueType())); - fields.add(DSL.field(MIN, metric.getValueType())); - fields.add(DSL.field(MAX, metric.getValueType())); - create.createTable(metric.getName()) - .columns(fields) - .execute(); - } - - public BatchBindStep startBatchPut(Metric metric, List dimensions) { - List dummyValues = new ArrayList<>(); - for (String dim: dimensions) { - dummyValues.add(null); - } - //Finally add sum, avg, min, max - dummyValues.add(null); - dummyValues.add(null); - dummyValues.add(null); - dummyValues.add(null); - return create.batch(create.insertInto(DSL.table(metric.getName())).values(dummyValues)); - } - - public BatchBindStep startBatchPut(String tableName, int dimNum) { - if (dimNum < 1 || !DBUtils.checkIfTableExists(create, tableName)) { - throw new IllegalArgumentException(String - .format("Incorrect arguments %s, %d", tableName, dimNum)); - } - List dummyValues = new ArrayList<>(dimNum); - for (int i = 0; i < dimNum; i++) { - dummyValues.add(null); - } - - return create.batch( - create.insertInto(DSL.table(tableName)).values(dummyValues)); - } - - public void putMetric(Metric metric, - Dimensions dimensions, - long windowStartTime) { - create.insertInto(DSL.table(metric.getName())) - .set(DSL.field(SUM, Double.class), metric.getSum()) - .set(DSL.field(AVG, Double.class), metric.getAvg()) - .set(DSL.field(MIN, Double.class), metric.getMin()) - .set(DSL.field(MAX, Double.class), metric.getMax()) - .set(dimensions.getFieldMap()) - .execute(); - } - - //We have a table per metric. We do a group by/aggregate on - //every dimension and return all the metric tables. - public List> getAggregatedMetricTables(List metrics, - List aggregations, List dimensions) throws Exception { - List> tList = new ArrayList<>(); - List> groupByFields = DBUtils.getFieldsFromList(dimensions); - - for (int i = 0; i < metrics.size(); i++) { - String metric = metrics.get(i); - List> selectFields = DBUtils.getFieldsFromList(dimensions); - String aggType = aggregations.get(i); - if (aggType.equals(SUM)) { - Field field = DSL.field(SUM, Double.class); - selectFields.add(DSL.sum(field).as(metric)); - } else if (aggType.equals(AVG)) { - Field field = DSL.field(AVG, Double.class); - selectFields.add(DSL.avg(field).as(metric)); - } else if (aggType.equals(MIN)) { - Field field = DSL.field(MIN, Double.class); - selectFields.add(DSL.min(field).as(metric)); - } else if (aggType.equals(MAX)) { - Field field = DSL.field(MAX, Double.class); - selectFields.add(DSL.max(field).as(metric)); - } else { - throw new Exception("Unknown agg type"); - } - if (!DBUtils.checkIfTableExists(create, metrics.get(i))) { - tList.add(null); - } else { - tList.add(create.select(selectFields) - .from(DSL.table(metric)) - .groupBy(groupByFields) - .asTable()); - } - } - return tList; - } - - - /** - * query metrics from different tables and merge to one table. - * - * getAggregatedMetricTables returns tables like: - * +-----+---------+-----+ - * |shard|indexName| cpu| - * +-----+---------+-----+ - * |0 |sonested | 10| - * |1 |sonested | 20| - * - * +-----+---------+-----+ - * |shard|indexName| rss| - * +-----+---------+-----+ - * |0 |sonested | 54| - * |2 |sonested | 47| - * - * We select metrics from each table and union them: - * +-----+---------+-----+-----+ - * |shard|indexName| cpu| rss| - * +-----+---------+-----+-----+ - * |0 |sonested | 10| null| - * |1 |sonested | 20| null| - * |0 |sonested | null| 54| - * |2 |sonested | null| 47| - * - * Then, we group by dimensions and return following table: - * +-----+---------+-----+-----+ - * |shard|indexName| cpu| rss| - * +-----+---------+-----+-----+ - * |0 |sonested | 10| 54| - * |1 |sonested | 20| null| - * |2 |sonested | null| 47| - * - * @param metrics a list of metrics we want to query - * @param aggregations aggregation we want to use for each metric - * @param dimensions dimension we want to use for each metric - * - * @return result of query - * - * @throws Exception if one of the aggregations contains sth other than - * "sum", "avg", "min", and "max". - * */ - public Result queryMetric(List metrics, - List aggregations, List dimensions) throws Exception { - List> tList = getAggregatedMetricTables(metrics, - aggregations, dimensions); - - //Join all the individual metric tables to generate the final table. - Select finalTable = null; - for (int i = 0; i < tList.size(); i++) { - TableLike metricTable = tList.get(i); - if (metricTable == null) { - LOG.info(String.format("%s metric table does not exist. " + - "Returning null for the metric/dimension.", metrics.get(i))); - continue; - } - List> selectFields = DBUtils.getSelectFieldsForMetricName(metrics.get(i), metrics, dimensions); - Select curTable = create.select(selectFields).from(metricTable); - - if (finalTable == null) { - finalTable = curTable; - } else { - finalTable = finalTable.union(curTable); - } - } - - List> allFields = DBUtils.getFieldsFromList(dimensions); - for (String metric : metrics) { - allFields.add(DSL.max(DSL.field(metric, Double.class)).as(metric)); - } - List> groupByFields = DBUtils.getFieldsFromList(dimensions); - if (finalTable == null) { - return null; - } - return create.select(allFields).from(finalTable).groupBy(groupByFields).fetch(); - } - - public void commit() throws Exception { - conn.commit(); - } - - @Override - public void remove() throws Exception { - conn.close(); - } - - public void deleteOnDiskFile() { - File dbFile = new File(getDBFilePath()); - if (!dbFile.delete()) { - LOG.error("Failed to delete File - {} with ExceptionCode: {}", - getDBFilePath(), StatExceptionCode.OTHER.toString()); - StatsCollector.instance().logException(); - } - } - - public Result queryMetric(String metric) { - return create.select().from(DSL.table(metric)).fetch(); - } - - public boolean metricExists(String metric) { - return DBUtils.checkIfTableExists(create, metric); - } -} - - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/model/MetricAttributes.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/model/MetricAttributes.java deleted file mode 100644 index e3fe67aa..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/model/MetricAttributes.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.model; - -import java.util.HashSet; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricDimension; - - -public class MetricAttributes { - public String unit; - public HashSet dimensionNames; - - MetricAttributes(String unit, - MetricDimension[] dimensions) { - - this.unit = unit; - this.dimensionNames = new HashSet(); - for (MetricDimension dimension : dimensions) { - this.dimensionNames.add(dimension.toString()); - } - } - -} \ No newline at end of file diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/model/MetricsModel.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/model/MetricsModel.java deleted file mode 100644 index e0d1bd13..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/model/MetricsModel.java +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.model; - -import java.util.HashMap; -import java.util.Map; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.AggregatedOSDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CircuitBreakerDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CircuitBreakerValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CommonMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.DiskDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.DiskValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.EmptyDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HeapDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HeapValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HttpOnlyDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HttpMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.LatencyDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricUnits; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.OSMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardBulkMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardOperationMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardStatsValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardStatsDerivedDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.TCPDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.TCPValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ThreadPoolDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ThreadPoolValue; - -public class MetricsModel { - - public static final Map ALL_METRICS = new HashMap<>(); - - static { - // OS Metrics - ALL_METRICS.put(OSMetrics.CPU_UTILIZATION.toString(), - new MetricAttributes(MetricUnits.CORES.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.PAGING_MAJ_FLT_RATE.toString(), - new MetricAttributes(MetricUnits.COUNT_PER_SEC.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.PAGING_MIN_FLT_RATE.toString(), - new MetricAttributes(MetricUnits.COUNT_PER_SEC.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.PAGING_RSS.toString(), - new MetricAttributes(MetricUnits.PAGES.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.SCHED_RUNTIME.toString(), - new MetricAttributes(MetricUnits.SEC_PER_CONTEXT_SWITCH.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.SCHED_WAITTIME.toString(), - new MetricAttributes(MetricUnits.SEC_PER_CONTEXT_SWITCH.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.SCHED_CTX_RATE.toString(), - new MetricAttributes(MetricUnits.COUNT_PER_SEC.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.HEAP_ALLOC_RATE.toString(), - new MetricAttributes(MetricUnits.BYTE_PER_SEC.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.IO_READ_THROUGHPUT.toString(), - new MetricAttributes(MetricUnits.BYTE_PER_SEC.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.IO_WRITE_THROUGHPUT.toString(), - new MetricAttributes(MetricUnits.BYTE_PER_SEC.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.IO_TOT_THROUGHPUT.toString(), - new MetricAttributes(MetricUnits.BYTE_PER_SEC.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.IO_READ_SYSCALL_RATE.toString(), - new MetricAttributes(MetricUnits.COUNT_PER_SEC.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.IO_WRITE_SYSCALL_RATE.toString(), - new MetricAttributes(MetricUnits.COUNT_PER_SEC.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.IO_TOTAL_SYSCALL_RATE.toString(), - new MetricAttributes(MetricUnits.COUNT_PER_SEC.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.THREAD_BLOCKED_TIME.toString(), - new MetricAttributes(MetricUnits.SEC_PER_EVENT.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(OSMetrics.THREAD_BLOCKED_EVENT.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), AggregatedOSDimension.values())); - - // Latency Metric - ALL_METRICS.put(CommonMetric.LATENCY.toString(), - new MetricAttributes(MetricUnits.MILLISECOND.toString(), LatencyDimension.values())); - - ALL_METRICS.put(ShardOperationMetric.SHARD_OP_COUNT.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), AggregatedOSDimension.values())); - ALL_METRICS.put(ShardBulkMetric.DOC_COUNT.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), AggregatedOSDimension.values())); - - // HTTP Metrics - ALL_METRICS.put(HttpMetric.HTTP_REQUEST_DOCS.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), HttpOnlyDimension.values())); - ALL_METRICS.put(HttpMetric.HTTP_TOTAL_REQUESTS.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), HttpOnlyDimension.values())); - - // Circuit Breaker Metrics - ALL_METRICS.put(CircuitBreakerValue.CB_ESTIMATED_SIZE.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), CircuitBreakerDimension.values())); - ALL_METRICS.put(CircuitBreakerValue.CB_CONFIGURED_SIZE.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), CircuitBreakerDimension.values())); - ALL_METRICS.put(CircuitBreakerValue.CB_TRIPPED_EVENTS.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), CircuitBreakerDimension.values())); - - // Heap Metrics - ALL_METRICS.put(HeapValue.GC_COLLECTION_EVENT.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), HeapDimension.values())); - ALL_METRICS.put(HeapValue.GC_COLLECTION_TIME.toString(), - new MetricAttributes(MetricUnits.MILLISECOND.toString(), HeapDimension.values())); - ALL_METRICS.put(HeapValue.HEAP_COMMITTED.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), HeapDimension.values())); - ALL_METRICS.put(HeapValue.HEAP_INIT.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), HeapDimension.values())); - ALL_METRICS.put(HeapValue.HEAP_MAX.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), HeapDimension.values())); - ALL_METRICS.put(HeapValue.HEAP_USED.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), HeapDimension.values())); - - // Disk Metrics - ALL_METRICS.put(DiskValue.DISK_UTILIZATION.toString(), - new MetricAttributes(MetricUnits.PERCENT.toString(), DiskDimension.values())); - ALL_METRICS.put(DiskValue.DISK_WAITTIME.toString(), - new MetricAttributes(MetricUnits.MILLISECOND.toString(), DiskDimension.values())); - ALL_METRICS.put(DiskValue.DISK_SERVICE_RATE.toString(), - new MetricAttributes(MetricUnits.MEGABYTE_PER_SEC.toString(), DiskDimension.values())); - - // TCP Metrics - ALL_METRICS.put(TCPValue.Net_TCP_NUM_FLOWS.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), TCPDimension.values())); - ALL_METRICS.put(TCPValue.Net_TCP_TXQ.toString(), - new MetricAttributes(MetricUnits.SEGMENT_PER_FLOW.toString(), TCPDimension.values())); - ALL_METRICS.put(TCPValue.Net_TCP_RXQ.toString(), - new MetricAttributes(MetricUnits.SEGMENT_PER_FLOW.toString(), TCPDimension.values())); - ALL_METRICS.put(TCPValue.Net_TCP_LOST.toString(), - new MetricAttributes(MetricUnits.SEGMENT_PER_FLOW.toString(), TCPDimension.values())); - ALL_METRICS.put(TCPValue.Net_TCP_SEND_CWND.toString(), - new MetricAttributes(MetricUnits.BYTE_PER_FLOW.toString(), TCPDimension.values())); - ALL_METRICS.put(TCPValue.Net_TCP_SSTHRESH.toString(), - new MetricAttributes(MetricUnits.BYTE_PER_FLOW.toString(), TCPDimension.values())); - - // IP Metrics - ALL_METRICS.put(IPValue.NET_PACKET_RATE4.toString(), - new MetricAttributes(MetricUnits.PACKET_PER_SEC.toString(), IPDimension.values())); - ALL_METRICS.put(IPValue.NET_PACKET_DROP_RATE4.toString(), - new MetricAttributes(MetricUnits.PACKET_PER_SEC.toString(), IPDimension.values())); - ALL_METRICS.put(IPValue.NET_PACKET_RATE6.toString(), - new MetricAttributes(MetricUnits.PACKET_PER_SEC.toString(), IPDimension.values())); - ALL_METRICS.put(IPValue.NET_PACKET_DROP_RATE6.toString(), - new MetricAttributes(MetricUnits.PACKET_PER_SEC.toString(), IPDimension.values())); - ALL_METRICS.put(IPValue.NET_THROUGHPUT.toString(), - new MetricAttributes(MetricUnits.BYTE_PER_SEC.toString(), IPDimension.values())); - - // Thread Pool Metrics - ALL_METRICS.put(ThreadPoolValue.THREADPOOL_QUEUE_SIZE.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ThreadPoolDimension.values())); - ALL_METRICS.put(ThreadPoolValue.THREADPOOL_REJECTED_REQS.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ThreadPoolDimension.values())); - ALL_METRICS.put(ThreadPoolValue.THREADPOOL_TOTAL_THREADS.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ThreadPoolDimension.values())); - ALL_METRICS.put(ThreadPoolValue.THREADPOOL_ACTIVE_THREADS.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ThreadPoolDimension.values())); - - // Shard Stats Metrics - ALL_METRICS.put(ShardStatsValue.INDEXING_THROTTLE_TIME.toString(), - new MetricAttributes(MetricUnits.MILLISECOND.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.CACHE_QUERY_HIT.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.CACHE_QUERY_MISS.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.CACHE_QUERY_SIZE.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.CACHE_FIELDDATA_EVICTION.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.CACHE_FIELDDATA_SIZE.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.CACHE_REQUEST_HIT.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.CACHE_REQUEST_MISS.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.CACHE_REQUEST_EVICTION.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.CACHE_REQUEST_SIZE.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.REFRESH_EVENT.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.REFRESH_TIME.toString(), - new MetricAttributes(MetricUnits.MILLISECOND.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.FLUSH_EVENT.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.FLUSH_TIME.toString(), - new MetricAttributes(MetricUnits.MILLISECOND.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.MERGE_EVENT.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.MERGE_TIME.toString(), - new MetricAttributes(MetricUnits.MILLISECOND.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.MERGE_CURRENT_EVENT.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.INDEXING_BUFFER.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.SEGMENTS_TOTAL.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.SEGMENTS_MEMORY.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.TERMS_MEMORY.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.STORED_FIELDS_MEMORY.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.TERM_VECTOR_MEMORY.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.NORMS_MEMORY.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.POINTS_MEMORY.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.DOC_VALUES_MEMORY.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.INDEX_WRITER_MEMORY.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.VERSION_MAP_MEMORY.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - ALL_METRICS.put(ShardStatsValue.BITSET_MEMORY.toString(), - new MetricAttributes(MetricUnits.BYTE.toString(), ShardStatsDerivedDimension.values())); - - // Master Metrics - ALL_METRICS.put(MasterPendingValue.MASTER_PENDING_QUEUE_SIZE.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), EmptyDimension.values())); - - ALL_METRICS.put(AllMetrics.MasterMetricValues.MASTER_TASK_QUEUE_TIME.toString(), - new MetricAttributes(MetricUnits.MILLISECOND.toString(), AllMetrics.MasterMetricDimensions.values())); - - ALL_METRICS.put(AllMetrics.MasterMetricValues.MASTER_TASK_RUN_TIME.toString(), - new MetricAttributes(MetricUnits.MILLISECOND.toString(), AllMetrics.MasterMetricDimensions.values())); - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/OSGlobals.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/OSGlobals.java deleted file mode 100644 index acbc1f2e..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/OSGlobals.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.os; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.logging.log4j.util.Supplier; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.ConfigStatus; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; - -public class OSGlobals { - private static long scClkTck; - private static String pid; - private static final String CLK_TCK_SYS_PROPERTY_NAME = "clk.tck"; - - private static final Logger LOGGER = LogManager.getLogger(OSGlobals.class); - private static final long REFRESH_INTERVAL_MS = MetricsConfiguration.CONFIG_MAP.get(OSGlobals.class).samplingInterval; - private static List tids = new ArrayList<>(); - private static long lastUpdated = -1; - - static { - try { - pid = new File("/proc/self").getCanonicalFile().getName(); - getScClkTckFromConfig(); - enumTids(); - lastUpdated = System.currentTimeMillis(); - } catch (Exception e) { - LOGGER.error( - (Supplier) () -> new ParameterizedMessage( - "Error in static initialization of OSGlobals with exception: {}", - e.toString()), - e); - } - } - - public static String getPid() { - return pid; - } - - public static long getScClkTck() { - return scClkTck; - } - - private static void getScClkTckFromConfig() throws Exception { - try { - scClkTck = Long.parseUnsignedLong(System.getProperty(CLK_TCK_SYS_PROPERTY_NAME)); - } catch (Exception e) { - LOGGER.error( - (Supplier) () -> new ParameterizedMessage( - "Error in reading/parsing clk.tck value: {}", - e.toString()), - e); - ConfigStatus.INSTANCE.setConfigurationInvalid(); - } - } - - private static void enumTids() { - tids.clear(); - tids.add(pid); - - File self = new File("/proc/self/task"); - File[] filesList = self.listFiles(); - for (File f : filesList) { - if (f.isDirectory()) { - String tid = f.getName(); - tids.add(tid); - } - } - } - - static synchronized List getTids() { - long curtime = System.currentTimeMillis(); - if (curtime - lastUpdated > REFRESH_INTERVAL_MS) { - enumTids(); - lastUpdated = curtime; - } - return new ArrayList<>(tids); - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/SchemaFileParser.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/SchemaFileParser.java deleted file mode 100644 index 5ebcc21d..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/SchemaFileParser.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.os; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.FileNotFoundException; -import java.util.HashMap; -import java.util.Map; -import java.util.List; -import java.util.ArrayList; -import java.util.Arrays; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; - -public class SchemaFileParser { - private static final Logger LOGGER = LogManager.getLogger(SchemaFileParser.class); - public enum FieldTypes { - INT, - STRING, - CHAR, - ULONG, - DOUBLE; - } - - private String filename = null; - private String[] keys = null; - private FieldTypes[] types = null; - private boolean preProcess = false; - - public SchemaFileParser (String file, String[] keys, FieldTypes[] types) { - this.filename = file; - this.keys = keys.clone(); - this.types = types.clone(); - } - - //- from java 11 onwards, there is thread name in /proc/pid/task/tid/stat, which has spaces in it - //- And threadname has "()" around it. Introduced a preprocess step to combine all of them - public SchemaFileParser (String file, String[] keys, FieldTypes[] types, boolean preProcess) { - this.filename = file; - this.keys = keys.clone(); - this.types = types.clone(); - this.preProcess = preProcess; - } - - private Object getTypedValue(String value, FieldTypes type) { - switch (type) { - case CHAR: - return value.charAt(0); - case INT: - return Integer.valueOf(value); - case STRING: - return value; - case ULONG: - return Long.parseUnsignedLong(value); - case DOUBLE: - return Double.valueOf(value); - default: - return null; - } - } - - private void generateMap(String content, Map map) { - String[] splitvalues = content.trim().split(" +"); - String[] values = preProcess(splitvalues); - if(values.length < types.length) { - LOGGER.debug("Content Values tokens {} length is less than types {} length with ExceptionCode: {}", - () -> Arrays.toString(values), () -> Arrays.toString(types), () -> StatExceptionCode.SCHEMA_PARSER_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.SCHEMA_PARSER_ERROR); - } - int lim = Math.min(values.length, types.length); - for (int idx = 0; idx < lim; idx++) { - map.put(keys[idx], getTypedValue(values[idx], types[idx])); - } - } - - private String[] preProcess(String[] tokens) { - if(preProcess) { - List processedTokens = new ArrayList<>(); - StringBuffer tmp = new StringBuffer(); - boolean beingProcessed = false; - for (int idx = 0; idx < tokens.length; idx++) { - if(beingProcessed) { - tmp.append(tokens[idx]); - if(tokens[idx].endsWith(")")) { - beingProcessed = false; - processedTokens.add(tmp.toString()); - tmp.setLength(0); - } - } else if(tokens[idx].startsWith("(")) { - if(tokens[idx].endsWith(")")) { - processedTokens.add(tokens[idx]); - } else { - beingProcessed = true; - tmp.append(tokens[idx]); - } - } else { - processedTokens.add(tokens[idx]); - } - } - return processedTokens.toArray(new String[processedTokens.size()]); - } else { - return tokens; - } - } - - /* - to be used for parsing the outputs that contains single line - */ - public Map parse() { - Map map = new HashMap<>(); - try (FileReader fileReader = new FileReader(new File(filename)); - BufferedReader bufferedReader = new BufferedReader(fileReader);) { - String line = bufferedReader.readLine(); - if (line == null) { - return map; - } - generateMap(line, map); - } catch (FileNotFoundException e) { - LOGGER.debug("FileNotFound in parse with exception: {}", () -> e.toString()); - } catch (Exception e) { - LOGGER.debug("Error in parse with exception: {} with ExceptionCode: {}", - () -> e.toString(), () -> StatExceptionCode.SCHEMA_PARSER_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.SCHEMA_PARSER_ERROR); - } - return map; - } - - /* - to be used for parsing the outputs that contains multiple lines - */ - public List> parseMultiple() { - List> mapList = new ArrayList<>(); - try (FileReader fileReader = new FileReader(new File(filename)); - BufferedReader bufferedReader = new BufferedReader(fileReader);) { - String line; - while ((line = bufferedReader.readLine()) != null) { - Map map = new HashMap<>(); - generateMap(line, map); - mapList.add(map); - } - } catch (FileNotFoundException e) { - LOGGER.debug("FileNotFound in parse with exception: {}", () -> e.toString()); - } catch (Exception e) { - LOGGER.debug("Error in parseMultiple with exception: {} with ExceptionCode: {}", - () -> e.toString(), () -> StatExceptionCode.SCHEMA_PARSER_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.SCHEMA_PARSER_ERROR); - } - return mapList; - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadCPU.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadCPU.java deleted file mode 100644 index fc169e34..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadCPU.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.os; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.logging.log4j.util.Supplier; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux.LinuxCPUPagingActivityGenerator; - -public final class ThreadCPU { - private static final Logger LOGGER = LogManager.getLogger(ThreadCPU.class); - public static final ThreadCPU INSTANCE = new ThreadCPU(); - private long scClkTck = 0; - private String pid = null; - private List tids = null; - private Map> tidKVMap = new HashMap<>(); - private Map> oldtidKVMap = new HashMap<>(); - private long kvTimestamp = 0; - private long oldkvTimestamp = 0; - private LinuxCPUPagingActivityGenerator cpuPagingActivityMap = new LinuxCPUPagingActivityGenerator(); - - // these two arrays map 1-1 - private static String[] statKeys = { - "pid", - "comm", - "state", - "ppid", - "pgrp", - "session", - "ttynr", - "tpgid", - "flags", - "minflt", - "cminflt", - "majflt", - "cmajflt", - "utime", - "stime", - "cutime", - "cstime", - "prio", - "nice", - "nthreads", - "itrealvalue", - "starttime", - "vsize", - "rss", - "rsslim", - "startcode", - "endcode", - "startstack", - "kstkesp", - "kstkeip", - "signal", - "blocked", - "sigignore", - "sigcatch", - "wchan", - "nswap", - "cnswap", - "exitsig", - "cpu", - "rtprio", - "schedpolicy", - "bio_ticks", - "vmtime", - "cvmtime" - // more that we ignore - }; - - private static SchemaFileParser.FieldTypes[] statTypes = { - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.STRING, - SchemaFileParser.FieldTypes.CHAR, - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.ULONG, //10 - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, //20 - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, //30 - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.INT, //40 - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.INT, - SchemaFileParser.FieldTypes.INT - }; - - private ThreadCPU() { - try { - pid = OSGlobals.getPid(); - scClkTck = OSGlobals.getScClkTck(); - tids = OSGlobals.getTids(); - } catch (Exception e) { - LOGGER.error( - (Supplier) () -> new ParameterizedMessage( - "Error In Initializing ThreadCPU: {}", - e.toString()), - e); - } - } - - public synchronized void addSample() { - tids = OSGlobals.getTids(); - - oldtidKVMap.clear(); - oldtidKVMap.putAll(tidKVMap); - - tidKVMap.clear(); - oldkvTimestamp = kvTimestamp; - kvTimestamp = System.currentTimeMillis(); - for (String tid : tids) { - Map sample = - //(new SchemaFileParser("/proc/"+tid+"/stat", - (new SchemaFileParser("/proc/" + pid + "/task/" + tid + "/stat", statKeys, statTypes, true)).parse(); - tidKVMap.put(tid, sample); - } - - calculateCPUDetails(); - calculatePagingActivity(); - } - - private void calculateCPUDetails() { - if (oldkvTimestamp == kvTimestamp) { - return; - } - - for (String tid : tidKVMap.keySet()) { - Map v = tidKVMap.get(tid); - Map oldv = oldtidKVMap.get(tid); - if (v != null && oldv != null) { - if (!v.containsKey("utime") || !oldv.containsKey("utime")) { - continue; - } - long diff = ((long) (v.getOrDefault("utime", 0L)) - (long) (oldv.getOrDefault("utime", 0L))) - + ((long) (v.getOrDefault("stime", 0L)) - (long) (oldv.getOrDefault("stime", 0L))); - double util = (1.0e3 * diff / scClkTck) / (kvTimestamp - oldkvTimestamp); - cpuPagingActivityMap.setCPUUtilization(tid, util); - } - } - } - - /** - * Note: major faults include mmap()'ed accesses - * - */ - private void calculatePagingActivity() { - if (oldkvTimestamp == kvTimestamp) { - return; - } - - - for (String tid : tidKVMap.keySet()) { - Map v = tidKVMap.get(tid); - Map oldv = oldtidKVMap.get(tid); - if (v != null && oldv != null) { - if (!v.containsKey("majflt") || !oldv.containsKey("majflt")) { - continue; - } - double majdiff = ((long) (v.getOrDefault("majflt", 0L)) - (long) (oldv.getOrDefault("majflt", 0L))); - majdiff /= 1.0e-3 * (kvTimestamp - oldkvTimestamp); - double mindiff = ((long) (v.getOrDefault("minflt", 0L)) - (long) (oldv.getOrDefault("minflt", 0L))); - mindiff /= 1.0e-3 * (kvTimestamp - oldkvTimestamp); - - Double[] fltarr = {majdiff, mindiff, (double) ((long) v.getOrDefault("rss", 0L))}; - cpuPagingActivityMap.setPagingActivities(tid, fltarr); - } - } - } - - public LinuxCPUPagingActivityGenerator getCPUPagingActivity() { - - return cpuPagingActivityMap; - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadDiskIO.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadDiskIO.java deleted file mode 100644 index 5915b6fd..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadDiskIO.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.os; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.FileNotFoundException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux.LinuxDiskIOMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class ThreadDiskIO { - private static String pid = OSGlobals.getPid(); - private static List tids = null; - private static final Logger LOGGER = LogManager.getLogger(ThreadDiskIO.class); - - private static Map> tidKVMap = new HashMap<>(); - private static Map> oldtidKVMap = new HashMap<>(); - private static long kvTimestamp = 0; - private static long oldkvTimestamp = 0; - - public static class IOMetrics { - public double avgReadThroughputBps; - public double avgWriteThroughputBps; - public double avgTotalThroughputBps; - - public double avgReadSyscallRate; - public double avgWriteSyscallRate; - public double avgTotalSyscallRate; - - public double avgPageCacheReadThroughputBps; - public double avgPageCacheWriteThroughputBps; - public double avgPageCacheTotalThroughputBps; - - @SuppressWarnings("checkstyle:parameternumber") - IOMetrics(double avgReadThroughputBps, - double avgReadSyscallRate, - double avgWriteThroughputBps, - double avgWriteSyscallRate, - double avgTotalThroughputBps, - double avgTotalSyscallRate, - double avgPageCacheReadThroughputBps, - double avgPageCacheWriteThroughputBps, - double avgPageCacheTotalThroughputBps) { - this.avgReadThroughputBps = avgReadThroughputBps; - this.avgWriteThroughputBps = avgWriteThroughputBps; - this.avgTotalThroughputBps = avgTotalThroughputBps; - this.avgReadSyscallRate = avgReadSyscallRate; - this.avgWriteSyscallRate = avgWriteSyscallRate; - this.avgTotalSyscallRate = avgTotalSyscallRate; - this.avgPageCacheReadThroughputBps = avgPageCacheReadThroughputBps; - this.avgPageCacheWriteThroughputBps = avgPageCacheWriteThroughputBps; - this.avgPageCacheTotalThroughputBps = avgPageCacheTotalThroughputBps; - } - public String toString() { - return new StringBuilder().append("rBps:").append(avgReadThroughputBps) - .append(" wBps:").append(avgWriteThroughputBps) - .append(" totBps:").append(avgTotalThroughputBps) - .append(" rSysc:").append(avgReadSyscallRate) - .append(" wSysc:").append(avgWriteSyscallRate) - .append(" totSysc:").append(avgTotalSyscallRate) - .append(" rPcBps:").append(avgPageCacheReadThroughputBps) - .append(" wPcBps:").append(avgPageCacheWriteThroughputBps) - .append(" totPcBps:").append(avgPageCacheTotalThroughputBps).toString(); - } - } - - - private static void addSampleTid(String tid) { - try (FileReader fileReader = new FileReader(new File("/proc/" + pid + "/task/" + tid + "/io")); - BufferedReader bufferedReader = new BufferedReader(fileReader);) { - String line = null; - Map kvmap = new HashMap<>(); - while ((line = bufferedReader.readLine()) != null) { - String[] toks = line.split("[: ]+"); - String key = toks[0]; - long val = Long.parseLong(toks[1]); - kvmap.put(key, val); - } - tidKVMap.put(tid, kvmap); - } catch (FileNotFoundException e) { - LOGGER.debug("FileNotFound in parse with exception: {}", () -> e.toString()); - } catch (Exception e) { - LOGGER.debug("Error In addSample Tid for: {} with error: {} with ExceptionCode: {}", - () -> tid, () -> e.toString(), () -> StatExceptionCode.THREAD_IO_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.THREAD_IO_ERROR); - } - } - - public static synchronized void addSample() { - tids = OSGlobals.getTids(); - oldtidKVMap.clear(); - oldtidKVMap.putAll(tidKVMap); - - tidKVMap.clear(); - oldkvTimestamp = kvTimestamp; - kvTimestamp = System.currentTimeMillis(); - for (String tid : tids) { - addSampleTid(tid); - } - } - - public static synchronized LinuxDiskIOMetricsGenerator getIOUtilization() { - - LinuxDiskIOMetricsGenerator linuxDiskIOMetricsHandler = new LinuxDiskIOMetricsGenerator(); - if (oldkvTimestamp == kvTimestamp) { - return linuxDiskIOMetricsHandler; - } - - for (String tid : tidKVMap.keySet()) { - Map v = tidKVMap.get(tid); - Map oldv = oldtidKVMap.get(tid); - if (v != null && oldv != null) { - double duration = 1.0e-3 * (kvTimestamp - oldkvTimestamp); - double readBytes = v.get("read_bytes") - oldv.get("read_bytes"); - double writeBytes = v.get("write_bytes") - oldv.get("write_bytes"); - double readSyscalls = v.get("syscr") - oldv.get("syscr"); - double writeSyscalls = v.get("syscw") - oldv.get("syscw"); - double readPcBytes = v.get("rchar") - oldv.get("rchar") - readBytes; - double writePcBytes = v.get("wchar") - oldv.get("wchar") - writeBytes; - readBytes /= duration; - readSyscalls /= duration; - writeBytes /= duration; - writeSyscalls /= duration; - readPcBytes /= duration; - writePcBytes /= duration; - - linuxDiskIOMetricsHandler.setDiskIOMetrics(tid, new IOMetrics(readBytes, readSyscalls, writeBytes, writeSyscalls, - readBytes+writeBytes, readSyscalls+writeSyscalls, - readPcBytes, writePcBytes, readPcBytes+writePcBytes)); - } - } - return linuxDiskIOMetricsHandler; - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadSched.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadSched.java deleted file mode 100644 index ea8d5f0b..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadSched.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.os; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.logging.log4j.util.Supplier; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.SchedMetricsGenerator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics_generator.linux.LinuxSchedMetricsGenerator; - -public final class ThreadSched { - private static final Logger LOGGER = LogManager.getLogger(ThreadSched.class); - public static final ThreadSched INSTANCE = new ThreadSched(); - private String pid = null; - private List tids = null; - private Map> tidKVMap = new HashMap<>(); - private Map> oldtidKVMap = new HashMap<>(); - private long kvTimestamp = 0; - private long oldkvTimestamp = 0; - - public static class SchedMetrics { - public final double avgRuntime; - public final double avgWaittime; - public final double contextSwitchRate; //both voluntary and involuntary - SchedMetrics(double avgRuntime, double avgWaittime, double contextSwitchRate) { - this.avgRuntime = avgRuntime; - this.avgWaittime = avgWaittime; - this.contextSwitchRate = contextSwitchRate; - } - @Override - public String toString() { - return new StringBuilder().append("avgruntime: ") - .append(avgRuntime).append(" avgwaittime: ").append(avgWaittime) - .append(" ctxrate: ").append(contextSwitchRate).toString(); - } - } - - private LinuxSchedMetricsGenerator schedLatencyMap = new LinuxSchedMetricsGenerator(); - - private static String[] schedKeys = { - "runticks", - "waitticks", - "totctxsws" - }; - - private static SchemaFileParser.FieldTypes[] schedTypes = { - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG, - SchemaFileParser.FieldTypes.ULONG - }; - - private ThreadSched() { - try { - pid = OSGlobals.getPid(); - tids = OSGlobals.getTids(); - } catch (Exception e) { - LOGGER.error( - (Supplier) () -> new ParameterizedMessage( - "Error In Initializing ThreadCPU: {}", - e.toString()), - e); - } - } - - public synchronized void addSample() { - tids = OSGlobals.getTids(); - - oldtidKVMap.clear(); - oldtidKVMap.putAll(tidKVMap); - - tidKVMap.clear(); - oldkvTimestamp = kvTimestamp; - kvTimestamp = System.currentTimeMillis(); - for (String tid : tids) { - Map sample = - (new SchemaFileParser("/proc/" + pid + "/task/" + tid + "/schedstat", - schedKeys, schedTypes)).parse(); - tidKVMap.put(tid, sample); - } - - calculateSchedLatency(); - } - - private void calculateSchedLatency() { - if (oldkvTimestamp == kvTimestamp) { - return; - } - - for (String tid : tidKVMap.keySet()) { - Map v = tidKVMap.get(tid); - Map oldv = oldtidKVMap.get(tid); - if (v != null && oldv != null) { - if (!v.containsKey("totctxsws") || !oldv.containsKey("totctxsws")) { - continue; - } - long ctxdiff = (long) v.getOrDefault("totctxsws", 0L) - (long) oldv.getOrDefault("totctxsws", 0L); - double avgRuntime = 1.0e-9 * ((long) v.getOrDefault("runticks", 0L) - (long) oldv.getOrDefault("runticks", 0L)); - double avgWaittime = 1.0e-9 * ((long) v.getOrDefault("waitticks", 0L) - (long) oldv.getOrDefault("waitticks", 0L)); - if (ctxdiff == 0) { - avgRuntime = 0; - avgWaittime = 0; - } else { - avgRuntime /= 1.0 * ctxdiff; - avgWaittime /= 1.0 * ctxdiff; - } - double contextSwitchRate = ctxdiff; - contextSwitchRate /= 1.0e-3 * (kvTimestamp - oldkvTimestamp); - - schedLatencyMap.setSchedMetric(tid, new SchedMetrics(avgRuntime, avgWaittime, contextSwitchRate)); - } - } - } - - public synchronized SchedMetricsGenerator getSchedLatency() { - - return schedLatencyMap; - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ClusterLevelMetricsReader.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ClusterLevelMetricsReader.java deleted file mode 100644 index 0564d42f..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ClusterLevelMetricsReader.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.NodeDetailColumns; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.JsonConverter; - -public class ClusterLevelMetricsReader { - - /** - * Almost the same as NodeDetailsCollector.NodeDetailsStatus. - * Consider keeping only one of them for easy maintenance. Don't do it now - * as we may separate reader and writer code later and we don't want many - * refactoring before release. - * - */ - public static class NodeDetails { - private String id; - private String hostAddress; - - NodeDetails(String stringifiedMetrics) { - Map map = JsonConverter - .createMapFrom(stringifiedMetrics); - id = (String) map.get(NodeDetailColumns.ID.toString()); - hostAddress = (String) map.get(NodeDetailColumns.HOST_ADDRESS - .toString()); - } - - @Override - public String toString() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("{"); - stringBuilder.append("id:" + id); - stringBuilder.append(" hostAddress:" + hostAddress); - stringBuilder.append("}"); - return stringBuilder.toString(); - } - - public String getId() { - return id; - } - - public String getHostAddress() { - return hostAddress; - } - } - - private static int sPollTimeInterval = 60000; - private static final Logger LOG = LogManager.getLogger(ClusterLevelMetricsReader.class); - private static int sBuckets = 60; - - private static NodeDetails[] nodesDetails = new NodeDetails[0]; - - public static NodeDetails[] getNodes() { - return nodesDetails.clone(); - } - - public static void collectNodeMetrics(long startTime) throws Exception { - String sNodesDetails = PerformanceAnalyzerMetrics.getMetric(startTime, PerformanceAnalyzerMetrics.sNodesPath); - - if(sNodesDetails != null) { - String lines[] = sNodesDetails.split("\\r?\\n"); - - - if(lines.length < 2) { - LOG.error("Skip parsing. Number of lines: {}.", lines.length); - return; - } - - NodeDetails[] tmpNodesDetails = new NodeDetails[lines.length-1]; - - // line 0 is last modified time of the file - - tmpNodesDetails[0] = new NodeDetails(lines[1]); - int tmpNodeDetailsIndex = 1; - - for(int i = 2; i < lines.length; i++) { - NodeDetails tmp = new NodeDetails(lines[i]); - - if(!tmp.id.equals(tmpNodesDetails[0].id)) { - tmpNodesDetails[tmpNodeDetailsIndex++] = tmp; - } - } - - nodesDetails = tmpNodesDetails; - } - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/FileHandler.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/FileHandler.java deleted file mode 100644 index 5fa0c97c..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/FileHandler.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.google.common.annotations.VisibleForTesting; - -public abstract class FileHandler { - private static final String[] EMPTY_STRING_ARRAY = new String[0]; - - private String rootLocation; - - // find all relevant files for a metric - public abstract List findFiles4Metric(long timeBucket); - - FileHandler() { - this.rootLocation = PerformanceAnalyzerMetrics.sDevShmLocation; - } - - public String[] processExtraDimensions(File file) throws IOException { - if (filePathRegex().isEmpty()) { - return EMPTY_STRING_ARRAY; - } - - // Note the question mark in the 1st group is reluctant - // quantifier. - Pattern pattern = Pattern.compile(filePathRegex()); - // our regex uses '/' as file separator - Matcher matcher = pattern.matcher(file.getCanonicalPath()); - if (matcher.find()) { - int groupCount = matcher.groupCount(); - String[] extraDimensions = new String[groupCount]; - // group 0 is the entire match - for (int i = 1; i <= groupCount; i++) { - extraDimensions[i-1] = matcher.group(i); - } - return extraDimensions; - } - throw new IOException(String.format( - "Cannot find a matching path %s", file.getCanonicalPath())); - - } - - // override this method if we need to extra dimensions from the file - // path - protected String filePathRegex() { - return ""; - } - - public String getRootLocation() { - return rootLocation; - } - - @VisibleForTesting - void setRootLocation(String location) { - rootLocation = location; - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/HttpRequestMetricsSnapshot.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/HttpRequestMetricsSnapshot.java deleted file mode 100644 index acf0d22f..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/HttpRequestMetricsSnapshot.java +++ /dev/null @@ -1,328 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import java.sql.Connection; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jooq.BatchBindStep; -import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.SQLDialect; -import org.jooq.SelectField; -import org.jooq.SelectHavingStep; -import org.jooq.impl.DSL; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.DBUtils; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CommonDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HttpDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HttpMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; - -/** - * Snapshot of start/end events generated by customer initiated http operations like bulk and search. - */ -@SuppressWarnings("serial") -public class HttpRequestMetricsSnapshot implements Removable { - private static final Logger LOG = LogManager.getLogger(HttpRequestMetricsSnapshot.class); - private static final Long EXPIRE_AFTER = 600000L; - private final DSLContext create; - private final Long windowStartTime; - private final String tableName; - private List columns; - - public enum Fields { - RID("rid"), - OPERATION(CommonDimension.OPERATION.toString()), - INDICES(HttpDimension.INDICES.toString()), - HTTP_RESP_CODE(HttpDimension.HTTP_RESP_CODE.toString()), - EXCEPTION(CommonDimension.EXCEPTION.toString()), - HTTP_REQUEST_DOCS(HttpMetric.HTTP_REQUEST_DOCS.toString()), - ST("st"), - ET("et"), - LAT("lat"), - HTTP_TOTAL_REQUESTS(HttpMetric.HTTP_TOTAL_REQUESTS.toString()); - - private final String fieldValue; - - Fields(String fieldValue) { - this.fieldValue = fieldValue; - } - - @Override - public String toString() { - return fieldValue; - } - } - - public HttpRequestMetricsSnapshot(Connection conn, Long windowStartTime) throws Exception { - this.create = DSL.using(conn, SQLDialect.SQLITE); - this.windowStartTime = windowStartTime; - this.tableName = "http_rq_" + windowStartTime; - - this.columns = new ArrayList() { { - this.add(Fields.RID.toString()); - this.add(Fields.OPERATION.toString()); - this.add(Fields.INDICES.toString()); - this.add(Fields.HTTP_RESP_CODE.toString()); - this.add(Fields.EXCEPTION.toString()); - this.add(Fields.HTTP_REQUEST_DOCS.toString()); - this.add(Fields.ST.toString()); - this.add(Fields.ET.toString()); - } }; - - List> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.RID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDICES.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.HTTP_RESP_CODE.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.EXCEPTION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.HTTP_REQUEST_DOCS.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.ST.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.ET.toString()), Long.class)); - } }; - - create.createTable(this.tableName) - .columns(fields) - .execute(); - } - - public void putStartMetric(Long startTime, Long itemCount, Map dimensions) { - Map, String> dimensionMap = new HashMap<>(); - for (Map.Entry dimension: dimensions.entrySet()) { - dimensionMap.put(DSL.field(DSL.name(dimension.getKey()), String.class), - dimension.getValue()); - } - create.insertInto(DSL.table(this.tableName)) - .set(DSL.field(DSL.name(Fields.ST.toString()), Long.class), startTime) - .set(DSL.field(DSL.name(Fields.HTTP_REQUEST_DOCS.toString()), Long.class), itemCount) - .set(dimensionMap) - .execute(); - } - - public BatchBindStep startBatchPut() { - List dummyValues = new ArrayList<>(); - for (int i = 0; i < columns.size(); i++) { - dummyValues.add(null); - } - return create.batch(create.insertInto(DSL.table(this.tableName)).values(dummyValues)); - } - - public void putEndMetric(Long endTime, Map dimensions) { - Map, String> dimensionMap = new HashMap<>(); - for (Map.Entry dimension: dimensions.entrySet()) { - dimensionMap.put(DSL.field( - DSL.name(dimension.getKey()), String.class), - dimension.getValue()); - } - create.insertInto(DSL.table(this.tableName)) - .set(DSL.field(DSL.name(Fields.ET.toString()), Long.class), endTime) - .set(dimensionMap) - .execute(); - } - - public Result fetchAll() { - return create.select().from(DSL.table(this.tableName)).fetch(); - } - - /** - * This function returns a single row for each request. - * We have a start and end event for each request and each event has different attributes. - * This function aggregates all the data into a single row. - * - * Actual Table - - * |rid |operation|indices |status|exception|itemCount| st| et| - * +-------+---------+--------+------+---------+---------+-------------+-------------+ - * |1417935|search | |{null}|{null} | 0|1535065254939| {null}| - * |1418424|search |{null} |200 | | {null}| {null}|1535065341025| - * |1418424|search |sonested|{null}|{null} | 0|1535065340730| {null}| - * |1418435|search |{null} |200 | | {null}| {null}|1535065343355| - * - * Returned Table - * |rid |operation|indices |status|exception|itemCount| st| et| - * +-------+---------+--------+------+---------+---------+-------------+-------------+ - * |1418424|search |sonested|200 | | 0|1535065340730|1535065341025| - * |1418435|search | |200 | | 0|1535065254939|1535065343355| - * - * @return a single row for each http request - */ - public SelectHavingStep groupByRidSelect() { - ArrayList> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.RID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - } }; - fields.add(DSL.max(DSL.field(Fields.ST.toString(), Long.class)).as(DSL.name(Fields.ST.toString()))); - fields.add(DSL.max(DSL.field(Fields.ET.toString(), Long.class)).as(DSL.name(Fields.ET.toString()))); - fields.add(DSL.max(DSL.field(Fields.INDICES.toString())).as(DSL.name(Fields.INDICES.toString()))); - fields.add(DSL.max(DSL.field(Fields.HTTP_RESP_CODE.toString())).as(DSL.name(Fields.HTTP_RESP_CODE.toString()))); - fields.add(DSL.max(DSL.field(Fields.EXCEPTION.toString())).as(DSL.name(Fields.EXCEPTION.toString()))); - fields.add(DSL.max(DSL.field(Fields.HTTP_REQUEST_DOCS.toString())).as(DSL.name(Fields.HTTP_REQUEST_DOCS.toString()))); - - return create.select(fields).from(DSL.table(this.tableName)) - .groupBy(DSL.field(Fields.RID.toString())); - } - - /** - * This function returns row with latency for each request. - * We have a start and end event for each request and each event has different attributes. - * This function aggregates all the data into a single row. - * - * Actual Table - - * |rid |operation|indices |status|exception|itemCount| st| et| - * +-------+---------+--------+------+---------+---------+-------------+-------------+ - * |1417935|search | |{null}|{null} | 0|1535065254939| {null}| - * |1418424|search |{null} |200 | | {null}| {null}|1535065341025| - * |1418424|search |sonested|{null}|{null} | 0|1535065340730| {null}| - * |1418435|search |{null} |200 | | {null}| {null}|1535065343355| - * - * Returned Table - * |rid |operation|indices |status|exception|itemCount| st| et| lat| - * +-------+---------+--------+------+---------+---------+-------------+-------------+-----+ - * |1418424|search |sonested|200 | | 0|1535065340730|1535065341025| 295| - * |1418435|search | |200 | | 0|1535065254939|1535065343355|88416| - * - * @return rows with latency for each request - */ - public SelectHavingStep fetchLatencyTable() { - ArrayList> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.RID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(Fields.ST.toString(), Long.class)); - this.add(DSL.field(Fields.ET.toString(), Long.class)); - this.add(DSL.field(Fields.HTTP_RESP_CODE.toString())); - this.add(DSL.field(Fields.INDICES.toString())); - this.add(DSL.field(Fields.EXCEPTION.toString())); - this.add(DSL.field(Fields.HTTP_REQUEST_DOCS.toString())); - } }; - fields.add(DSL.field(Fields.ET.toString()).minus(DSL.field(Fields.ST.toString())).as(DSL.name(Fields.LAT.toString()))); - return create.select(fields).from(groupByRidSelect()) - .where(DSL.field(Fields.ET.toString()).isNotNull().and( - DSL.field(Fields.ST.toString()).isNotNull())); - } - - /** - * This function aggregates rows by operation. - * This is a performance optimization to avoid writing one entry per request back into metricsDB. - * This function returns one row per operation. - * - * Latency Table - - * |rid |operation|indices |status|exception|itemCount| st| et|lat| - * +-------+---------+--------+------+---------+---------+-------------+-------------+-----+ - * |1418424|search |sonested|200 | | 0|1535065340730|1535065341025|295| - * |1418435|search |sonested|200 | | 0|1535065254939|1535065343355|305| - * - * Returned Table - - * |operation|indices |status|exception|sum_lat|avg_lat|min_lat|max_lat| - * +---------+--------+------+---------+---------+-------------+-------+ - * |search |sonested|200 | | 600| 300| 295| 305| - * - * @return latency rows by operation - */ - public Result fetchLatencyByOp() { - ArrayList> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.HTTP_RESP_CODE.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDICES.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.EXCEPTION.toString()), String.class)); - this.add(DSL.sum(DSL.field(DSL.name(Fields.HTTP_REQUEST_DOCS.toString()), Long.class)) - .as(DBUtils.getAggFieldName(Fields.HTTP_REQUEST_DOCS.toString(), MetricsDB.SUM))); - this.add(DSL.avg(DSL.field(DSL.name(Fields.HTTP_REQUEST_DOCS.toString()), Long.class)) - .as(DBUtils.getAggFieldName(Fields.HTTP_REQUEST_DOCS.toString(), MetricsDB.AVG))); - this.add(DSL.min(DSL.field(DSL.name(Fields.HTTP_REQUEST_DOCS.toString()), Long.class)) - .as(DBUtils.getAggFieldName(Fields.HTTP_REQUEST_DOCS.toString(), MetricsDB.MIN))); - this.add(DSL.max(DSL.field(DSL.name(Fields.HTTP_REQUEST_DOCS.toString()), Long.class)) - .as(DBUtils.getAggFieldName(Fields.HTTP_REQUEST_DOCS.toString(), MetricsDB.MAX))); - this.add(DSL.sum(DSL.field(DSL.name(Fields.LAT.toString()), Double.class)) - .as(DBUtils.getAggFieldName(Fields.LAT.toString(), MetricsDB.SUM))); - this.add(DSL.avg(DSL.field(DSL.name(Fields.LAT.toString()), Double.class)) - .as(DBUtils.getAggFieldName(Fields.LAT.toString(), MetricsDB.AVG))); - this.add(DSL.min(DSL.field(DSL.name(Fields.LAT.toString()), Double.class)) - .as(DBUtils.getAggFieldName(Fields.LAT.toString(), MetricsDB.MIN))); - this.add(DSL.max(DSL.field(DSL.name(Fields.LAT.toString()), Double.class)) - .as(DBUtils.getAggFieldName(Fields.LAT.toString(), MetricsDB.MAX))); - this.add(DSL.count().as(Fields.HTTP_TOTAL_REQUESTS.toString())); - } }; - ArrayList> groupByFields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.HTTP_RESP_CODE.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDICES.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.EXCEPTION.toString()), String.class)); - } }; - - return create.select(fields).from(fetchLatencyTable()) - .groupBy(groupByFields).fetch(); - } - - /** - * This function returns requests with a missing end event. - * A request maybe long running and the end event might not have occured in this snapshot. - * - * Actual Table - - * |rid |operation|indices |status|exception|itemCount| st| et| - * +-------+---------+--------+------+---------+---------+-------------+-------------+ - * |1417935|search | |{null}|{null} | 0|1535065254939| {null}| - * |1418424|search |sonested|{null}|{null} | 0|1535065340730| {null}| - * |1418435|search |{null} |200 | | {null}| {null}|1535065343355| - * - * Returned Table - * |rid |operation|indices |status|exception|itemCount| st| et| - * +-------+---------+--------+------+---------+---------+-------------+-------------+ - * |1418424|search |sonested|200 | | 0|1535065340730| | - * - * @return rows missing an end event - */ - public SelectHavingStep fetchInflightRequests() { - ArrayList> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.RID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDICES.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.HTTP_RESP_CODE.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.EXCEPTION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.HTTP_REQUEST_DOCS.toString()), Long.class)); - this.add(DSL.field(Fields.ST.toString(), Long.class)); - this.add(DSL.field(Fields.ET.toString(), Long.class)); - } }; - - return create.select(fields).from(groupByRidSelect()) - .where(DSL.field(Fields.ST.toString()).isNotNull() - .and(DSL.field(Fields.ET.toString()).isNull()) - .and(DSL.field(Fields.ST.toString()).gt(this.windowStartTime - EXPIRE_AFTER))); - } - - public String getTableName() { - return this.tableName; - } - - @Override - public void remove() { - LOG.info("Dropping table - {}", this.tableName); - create.dropTable(DSL.table(this.tableName)).execute(); - } - - public void rolloverInflightRequests(HttpRequestMetricsSnapshot prevSnap) { - //Fetch all entries that have not ended and write to current table. - create.insertInto(DSL.table(this.tableName)).select( - create.select().from(prevSnap.fetchInflightRequests())).execute(); - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MasterEventMetricsSnapshot.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MasterEventMetricsSnapshot.java deleted file mode 100644 index 644fc637..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MasterEventMetricsSnapshot.java +++ /dev/null @@ -1,311 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.DBUtils; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jooq.BatchBindStep; -import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.SQLDialect; -import org.jooq.SelectField; -import org.jooq.SelectHavingStep; -import org.jooq.impl.DSL; - - -import java.sql.Connection; -import java.util.ArrayList; -import java.util.List; - - -public class MasterEventMetricsSnapshot implements Removable { - private static final Logger LOG = LogManager.getLogger(MasterEventMetricsSnapshot.class); - - private final DSLContext create; - private final Long windowStartTime; - private final String tableName; - private static final Long EXPIRE_AFTER = 1200000L; - private List> columns; - - - public enum Fields { - - TID("tid"), - IS_CURRENT("isCurrent"), - OLD_START("oldStart"), - ST("st"), - ET("et"), - LAT ("lat"); - - private final String fieldValue; - - Fields(String fieldValue) { - this.fieldValue = fieldValue; - } - - @Override - public String toString() { - return fieldValue; - } - }; - - public MasterEventMetricsSnapshot(Connection conn, Long windowStartTime) { - this.create = DSL.using(conn, SQLDialect.SQLITE); - this.windowStartTime = windowStartTime; - this.tableName = "master_event_" + windowStartTime; - - this.columns = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.TID.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_INSERT_ORDER.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.ST.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.ET.toString()), Long.class)); - } }; - - create.createTable(this.tableName) - .columns(columns) - .execute(); - } - - - @Override - public void remove() throws Exception { - - create.dropTable(DSL.table(this.tableName)).execute(); - } - - public void rolloverInflightRequests(MasterEventMetricsSnapshot prevSnap) { - //Fetch all entries that have not ended and write to current table. - create.insertInto(DSL.table(this.tableName)).select(prevSnap.fetchInflightRequests()).execute(); - - LOG.debug("Inflight shard requests"); - LOG.debug(() -> fetchAll()); - } - - private SelectHavingStep fetchInflightRequests() { - - ArrayList> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.TID.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_INSERT_ORDER.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.ST.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.ET.toString()), Long.class)); - } }; - - return create.select(fields).from(groupByInsertOrder()) - .where(DSL.field(Fields.ST.toString()).isNotNull() - .and(DSL.field(Fields.ET.toString()).isNull()) - .and(DSL.field(Fields.ST.toString()).gt(this.windowStartTime - EXPIRE_AFTER))); - } - - - /** - * Return all master task event in the current window. - * - * Actual Table - * |tid |insertOrder|taskType |priority|queueTime|metadata| st| et| - * +-----+-----------+------------+--------+---------+--------+-------------+-------------+ - * |111 |1 |create-index|urgent |3 |{string}|1535065340625| {null}| - * |111 |2 |create-index|urgent |12 |{string}|1535065340825| {null}| - * |111 |1 | {null}| {null}| {null}| {null}| {null}|1535065340725| - * - * @return aggregated master task - */ - public Result fetchAll() { - - return create.select().from(DSL.table(this.tableName)).fetch(); - } - - public BatchBindStep startBatchPut() { - - List dummyValues = new ArrayList<>(); - for (int i = 0; i < columns.size(); i++) { - dummyValues.add(null); - } - return create.batch(create.insertInto(DSL.table(this.tableName)).values(dummyValues)); - } - - - /** - * Return one row per master task event. Group by the InsertOrder. - * It has 12 columns - * |InsertOrder|Priority|Type|Metadata|SUM_QueueTime|AVG_QueueTime|MIN_QueueTime|MAX_QueueTime| - * SUM_RUNTIME|AVG_RUNTIME|MIN_RUNTIME|MAX_RUNTIME| - * - * @return aggregated master task - */ - public Result fetchQueueAndRunTime() { - - List> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_INSERT_ORDER.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString()), String.class)); - - this.add(DSL.sum(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString()), Double.class)) - .as(DBUtils.getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString(), MetricsDB.SUM))); - this.add(DSL.avg(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString()), Double.class)) - .as(DBUtils.getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString(), MetricsDB.AVG))); - this.add(DSL.min(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString()), Double.class)) - .as(DBUtils.getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString(), MetricsDB.MIN))); - this.add(DSL.max(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString()), Double.class)) - .as(DBUtils.getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString(), MetricsDB.MAX))); - - this.add(DSL.sum(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString()), Double.class)) - .as(DBUtils.getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString(), MetricsDB.SUM))); - this.add(DSL.avg(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString()), Double.class)) - .as(DBUtils.getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString(), MetricsDB.AVG))); - this.add(DSL.min(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString()), Double.class)) - .as(DBUtils.getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString(), MetricsDB.MIN))); - this.add(DSL.max(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString()), Double.class)) - .as(DBUtils.getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString(), MetricsDB.MAX))); - } }; - - ArrayList> groupByFields = new ArrayList>() { { - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_INSERT_ORDER.toString()), String.class)); - } }; - - return create.select(fields).from(fetchRunTimeHelper()) - .groupBy(groupByFields) - .fetch(); - } - - private SelectHavingStep fetchRunTimeHelper() { - - List> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_INSERT_ORDER.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString()), String.class)); - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString()), String.class)); - this.add(DSL.field(Fields.ET.toString()).minus(DSL.field(Fields.ST.toString())). - as(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString()))); - } }; - - return create.select(fields).from(groupByInsertOrderAndAutoFillEndTime()) - .where(DSL.field(Fields.ET.toString()).isNotNull().and( - DSL.field(Fields.ST.toString()).isNotNull())); - } - - - /** - * Return one row per master task event. Group by the InsertOrder. - * For a master task without a finish event, we will use the current window end time - * - * CurrentWindowEndTime: 1535065341025 - * Actual Table - * |tid |insertOrder|taskType |priority|queueTime|metadata| st| et| - * +-----+-----------+------------+--------+---------+--------+-------------+-------------+ - * |111 |1 |create-index|urgent |3 |{string}|1535065340625| {null}| - * |111 |2 |create-index|urgent |12 |{string}|1535065340825| {null}| - * |111 |1 | {null}| {null}| {null}| {null}| {null}|1535065340725| - * - * Returned: - * - * |tid |insertOrder|taskType |priority|queueTime|metadata| st| et| - * +-----+-----------+------------+--------+---------+--------+-------------+-------------+ - * |111 |1 |create-index|urgent |3 |{string}|1535065340625|1535065340725| - * |111 |2 |create-index|urgent |12 |{string}|1535065340825|1535065341025| - * - * @return aggregated master task - */ - private SelectHavingStep groupByInsertOrderAndAutoFillEndTime() { - - Long endTime = windowStartTime + MetricsConfiguration.SAMPLING_INTERVAL; - List> fields = getGroupByInsertOrderSelectFields(); - fields.add(DSL.least(DSL.coalesce(DSL.max(DSL.field(Fields.ET.toString(), Long.class)), endTime), endTime) - .as(DSL.name(Fields.ET.toString()))); - - ArrayList> groupByInsertOrder = new ArrayList>() { { - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_INSERT_ORDER.toString()), String.class)); - } }; - - return create.select(fields) - .from(DSL.table(this.tableName)) - .groupBy(groupByInsertOrder); - } - - /** - * Return one row per master task event. Group by the InsertOrder, with possible et remains as null - * - * Actual Table - * |tid |insertOrder|taskType |priority|queueTime|metadata| st| et| - * +-----+-----------+------------+--------+---------+--------+-------------+-------------+ - * |111 |1 |create-index|urgent |3 |{string}|1535065340625| {null}| - * |111 |2 |create-index|urgent |12 |{string}|1535065340825| {null}| - * |111 |1 | {null}| {null}| {null}| {null}| {null}|1535065340725| - * - * Returned: - * - * |tid |insertOrder|taskType |priority|queueTime|metadata| st| et| - * +-----+-----------+------------+--------+---------+--------+-------------+-------------+ - * |111 |1 |create-index|urgent |3 |{string}|1535065340625|1535065340725| - * |111 |2 |create-index|urgent |12 |{string}|1535065340825| {null}| - * - * @return aggregated latency rows for each shard request - */ - private SelectHavingStep groupByInsertOrder() { - - ArrayList> fields = getGroupByInsertOrderSelectFields(); - - fields.add(DSL.max(DSL.field(Fields.ET.toString(), Long.class)).as(DSL.name(Fields.ET.toString()))); - fields.add(DSL.field(DSL.name(Fields.TID.toString()), String.class)); - - ArrayList> groupByInsertOrder = new ArrayList>() { { - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_INSERT_ORDER.toString()), String.class)); - } }; - - return create.select(fields) - .from(DSL.table(this.tableName)) - .groupBy(groupByInsertOrder); - } - - private ArrayList> getGroupByInsertOrderSelectFields() { - - ArrayList> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_INSERT_ORDER.toString()), String.class)); - - this.add(DSL.max(DSL.field(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString())) - .as(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString()))); - - this.add(DSL.max(DSL.field(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString())) - .as(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString()))); - - this.add(DSL.max(DSL.field(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString())) - .as(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString()))); - - this.add(DSL.max(DSL.field(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString())) - .as(DSL.name(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString()))); - - this.add(DSL.max(DSL.field(Fields.ST.toString(), Long.class)).as(DSL.name(Fields.ST.toString()))); - - } }; - - return fields; - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MemoryDBSnapshot.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MemoryDBSnapshot.java deleted file mode 100644 index fce168db..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MemoryDBSnapshot.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import java.sql.Connection; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jooq.BatchBindStep; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Record1; -import org.jooq.Result; -import org.jooq.SQLDialect; -import org.jooq.SelectField; -import org.jooq.SelectHavingStep; -import org.jooq.SelectJoinStep; -import org.jooq.impl.DSL; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.DBUtils; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricName; - -public class MemoryDBSnapshot implements Removable { - private static final Logger LOG = LogManager - .getLogger(OSMetricsSnapshot.class); - - private static final String WEIGHT = "weight"; - - private static final Field WEIGHT_FIELD = - DSL.field(WEIGHT, Double.class); - - protected final DSLContext create; - protected final String tableName; - - // the last update time of the /dev/shm/performanceanalyzer file that is the data - // source of our db table. - protected long lastUpdatedTime; - - private final List dimensionNames; - - private final List> dimensionsFields; - - private final List> metadata; - - // We have 1 table for every metadata in the disk database. This map stores - // the mapping from table names to the fields used to retrieve table - // contents from memory db to disk db. - private final Map>> tableSelectFieldsMap; - - private final Map>> tableGroupByFieldsMap; - - private final Map tableWhereClauseMap; - - private final boolean isAligned; - - /** - * - * @param conn In-memory database connection - * @param tableNamePrefix db table name prefix - * @param windowEndTime - * When creating un-aligned db snapshot, we use the time stamp - * when reader starts processing in a round as the 3rd parameter - * (Let's call it readerProcessTime). Reader process - * /dev/shm/performanceanalyzer files every 5 seconds. When creating aligned db - * snapshot, we use previous window's end time. Note the previous - * window's end time is computed after using - * PerformanceAnalyzerMetrics.getTimeInterval(). So if previous window's actual - * end time is 6000. After invoking - * PerformanceAnalyzerMetrics.getTimeInterval(6000, - * MetricsConfiguration.SAMPLING_INTERVAL), it is 5000. - * @param aligned whether this snapshot is for aligning - */ - public MemoryDBSnapshot(Connection conn, MetricName tableNamePrefix, - long windowEndTime, boolean aligned) { - this.create = DSL.using(conn, SQLDialect.SQLITE); - this.isAligned = aligned; - String tableNameSuffix = aligned ? "_aligned" : ""; - this.tableName = tableNamePrefix.toString() + windowEndTime - + tableNameSuffix; - lastUpdatedTime = -1; - - dimensionNames = MetricPropertiesConfig.getInstance() - .getProperty(tableNamePrefix) - .getDimensionNames(); - dimensionsFields = MetricPropertiesConfig.getInstance() - .getProperty(tableNamePrefix) - .getDimensionFields(); - metadata = MetricPropertiesConfig.getInstance() - .getProperty(tableNamePrefix) - .getMetricFields(); - - tableSelectFieldsMap = MetricPropertiesConfig.getInstance() - .getProperty(tableNamePrefix).getTableSelectMap(); - tableGroupByFieldsMap = MetricPropertiesConfig.getInstance() - .getProperty(tableNamePrefix).getTableGroupByFieldsMap(); - tableWhereClauseMap = MetricPropertiesConfig.getInstance() - .getProperty(tableNamePrefix).getTableWhereClauseMap(); - - // the tables should have columns in order: - // dimensions columns, metrics columns - LOG.debug("Creating a new snapshot table - {}", tableName); - create.createTable(this.tableName) - .columns(dimensionsFields) - .columns(metadata).execute(); - } - - public MemoryDBSnapshot(Connection conn, MetricName tableNamePrefix, - long windowEndTime) { - this(conn, tableNamePrefix, windowEndTime, false); - } - - public DSLContext getDSLContext() { - return create; - } - - @Override - public void remove() { - LOG.info("Dropping {}", this.tableName); - if (dbTableExists()) { - create.dropTable(DSL.table(this.tableName)).execute(); - } - } - - /** - * - * @return the last update time of the /dev/shm/performanceanalyzer file that is the data - * source of our db table. - */ - public long getLastUpdatedTime() { - return this.lastUpdatedTime; - } - - public void setLastUpdatedTime(long lastUpdatedTime) { - this.lastUpdatedTime = lastUpdatedTime; - } - - public String getTableName() { - return this.tableName; - } - - public SelectHavingStep selectAll() { - return create.select().from(this.tableName); - } - - public Result fetchAll() { - return create.select().from(DSL.table(this.tableName)).fetch(); - } - - public BatchBindStep startBatchPut() { - int totalSize = this.dimensionsFields.size() + this.metadata.size(); - List dummyValues = new ArrayList<>(totalSize); - for (int i = 0; i < this.dimensionsFields.size(); i++) { - dummyValues.add(null); - } - for (int i = 0; i < this.metadata.size(); i++) { - dummyValues.add(null); - } - return create.batch(create.insertInto(DSL.table(this.tableName)) - .values(dummyValues)); - } - - public List> getDimensions() { - return dimensionsFields; - } - - public Collection> getMetrics() { - return metadata; - } - - /** - * Given metrics in two windows calculates a new window which overlaps with the given windows. - * |------leftWindow-------|-------rightWindow--------| - * t - * a b - * |-----------alignedWindow------| - * - * This method assumes that both left/right windows are greater than or - * equal to 5 seconds. - * - * @param leftWindow - * MemoryDBSnapshot for the /dev/shm/performanceanalyzer file written before t. - * We save MemoryDBSnapshot in a map where the key is the time at - * which the /dev/shm/performanceanalyzer file was written, and the value is the - * MemoryDBSnapshot itself. - * @param rightWindow - * MemoryDBSnapshot for the /dev/shm/performanceanalyzer file written after t - * @param a - * aligned window start time. - * @param b - * aligned window end time. - * @param t - * leftWindow end time, as well as right window start time - */ - public void alignWindow(MemoryDBSnapshot leftWindow, - MemoryDBSnapshot rightWindow, long t, long a, - long b) { - ArrayList> alignedFields = - new ArrayList>(); - alignedFields.addAll(getDimensions()); - for (Field metric : getMetrics()) { - alignedFields.add(DSL.sum(metric) - .div(DSL.sum(WEIGHT_FIELD)) - .as(metric)); - } - - List> leftWinFields = new ArrayList>(); - leftWinFields.addAll(getDimensions()); - leftWinFields.add(DSL.val(t - a).as(WEIGHT)); - for (Field metric : getMetrics()) { - leftWinFields.add(metric.mul(t - a).as(metric.getName())); - } - List> rightWinFields = new ArrayList>(); - rightWinFields.addAll(getDimensions()); - rightWinFields.add(DSL.val(b - t).as(WEIGHT)); - for (Field metric : getMetrics()) { - rightWinFields.add(metric.mul(b - t).as(metric.getName())); - } - - SelectJoinStep recordsSource = create - .select(alignedFields) - .from(create.select(leftWinFields).from(leftWindow.tableName) - .unionAll(create.select(rightWinFields) - .from(rightWindow.getTableName()))); - - if (getDimensions().isEmpty()) { - create.insertInto(DSL.table(this.tableName)).select(recordsSource) - .execute(); - } else { - create.insertInto(DSL.table(this.tableName)) - .select(recordsSource.groupBy(getDimensions())).execute(); - } - } - - /** - * Precondition: The order of columns in each values[i] should match the - * table we have created in the constructor. We cannot check this - * programmatically. People who write code calling this method is - * responsible for verification. - * - * @param values - * each values[i] is a row - */ - public void insertMultiRows(Object[][] values) { - if (values == null || values.length == 0) { - return; - } - BatchBindStep batchHandle = startBatchPut(); - for (int i = 0; i < values.length; i++) { - batchHandle.bind(values[i]); - } - batchHandle.execute(); - } - - protected Result fetchMetric( - Condition condition, SelectField... column) { - return create.select(column) - .from(DSL.table(this.tableName)).where(condition).fetch(); - } - - protected boolean dbTableExists() { - return DBUtils.checkIfTableExists(create, tableName); - } - - protected Result> fetchTableSchema() { - return create.select(DSL.field("sql", String.class)) - .from(DSL.table("sqlite_master")) - .where(DSL.field("name", String.class).eq(this.tableName)) - .fetch(); - } - - public Map> selectMetadataSource() { - Map> selectFromTable = new HashMap<>(); - for (Map.Entry>> entry : tableSelectFieldsMap - .entrySet()) { - String tableName = entry.getKey(); - selectFromTable.put(tableName, - create.select(entry.getValue()).from(this.tableName) - .where(tableWhereClauseMap.get(tableName)) - .groupBy(tableGroupByFieldsMap.get(tableName))); - } - return selectFromTable; - } - - public Map>> getTableSelectFieldsMap() { - return tableSelectFieldsMap; - } - - public List getDimensionNames() { - return dimensionNames; - } - - public boolean isAligned() { - return isAligned; - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricProperties.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricProperties.java deleted file mode 100644 index c5af2058..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricProperties.java +++ /dev/null @@ -1,365 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jooq.BatchBindStep; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.impl.DSL; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.DBUtils; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.JsonConverter; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.JsonPathNotFoundException; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.annotations.VisibleForTesting; - -public class MetricProperties { - private static final Logger LOG = LogManager.getLogger(MetricProperties.class); - - - public static final MetricDimension[] EMPTY_DIMENSION = - new MetricDimension[] {}; - - private FileHandler handler; - // dimensions inferred else where (e.g., index name in file path). - // The order should match the grouping parts in filePathRegex. For example, - // if index name is the first group, and shard is the 2nd group, the first - // element of derivedDimension should be index name and the 2nd element - // should be shard id. - private MetricDimension[] derivedDimension; - private MetricDimension[] directDimensions; - private MetricValue[] metadata; - - // a list of dimension names - private List dimensionNames; - - // a list of dimension fields derived from dimensionNames - private List> dimensionFields; - - // map from table prefix name to a list of metadata fields - private List> metadataFields; - - // We have 1 table for every metadata in the disk database. This list stores - // table names for each metadata in metadata in order. Usually, we use - // metadata name for table name. But it is possible we use the same medatada - // name in different snapshots. For example, "util" can be in both - // disk/network (not true right now). - // We make it configurable on our end, so that we don't run into this - // issue. - private List metadataTableNames; - - // disk db table name -> fields in select from memory db table to get - // contents for the disk db table - private Map>> inMemoryTableSelectFieldsMap = - new HashMap<>(); - - private Map>> inMemoryTableGroupByFieldsMap = - new HashMap<>(); - - private Map inMemoryTableWhereClauseMap = - new HashMap<>(); - - public MetricProperties( - MetricDimension[] derivedDimension, MetricDimension[] dimensions, - MetricValue[] values, FileHandler handler, - Map customizedTableNames) { - this(derivedDimension, dimensions, values, handler); - customizeMetricTableName(customizedTableNames); - - initializeTableSelectFields(); - } - - public MetricProperties( - MetricDimension[] derivedDimension, MetricDimension[] dimensions, - MetricValue[] values, FileHandler handler) { - super(); - this.handler = handler; - this.derivedDimension = derivedDimension.clone(); - this.directDimensions = dimensions.clone(); - this.metadata = values.clone(); - this.inMemoryTableSelectFieldsMap = new HashMap<>(); - - initializeFields(); - - initializeTableSelectFields(); - } - - public MetricProperties( - MetricDimension[] dimensions, - MetricValue[] values, FileHandler handler) { - this(EMPTY_DIMENSION, dimensions, values, handler); - } - - public List> getMetricFields() { - return metadataFields; - } - - public List> getDimensionFields() { - return dimensionFields; - } - - @VisibleForTesting - void setHandler(FileHandler handler) { - this.handler = handler; - } - - @VisibleForTesting - FileHandler getHandler() { - return handler; - } - - boolean processMetrics(File file, MemoryDBSnapshot snap, long startTime, - long lastSnapTimestamp, BatchBindStep batchHandle) - throws IOException { - - try (BufferedReader bufferedReader = new BufferedReader( - new FileReader(file))) { - String line = bufferedReader.readLine(); - if (line == null) { - return false; - } - - long lastModifiedTime = JsonConverter.getLongValue(line, - PerformanceAnalyzerMetrics.METRIC_CURRENT_TIME); - - // Only consider metrics if the file has been updated in the 5 - // second window. - if (lastModifiedTime > startTime - || lastModifiedTime <= lastSnapTimestamp) { - return false; - } - - // snap's last updated time is the highest last modified time of all - // the entries in the snapshot. - if (snap.getLastUpdatedTime() < lastModifiedTime) { - snap.setLastUpdatedTime(lastModifiedTime); - } - - String[] derivedDimension = handler.processExtraDimensions(file); - - int numMetrics = derivedDimension.length - + directDimensions.length + metadata.length; - Object[] templateMetricVals = new Object[numMetrics]; - int valIndex = 0; - - for (int i = 0; i < derivedDimension.length; i++) { - templateMetricVals[valIndex++] = derivedDimension[i]; - } - - boolean processed = false; - // first line is last modified time of the file. - // We need last modified time in milliseconds. But JDK method - // File.lastModified() cannot give that precision. So we need - // to add last modified time by ourselves. - // See: - // https://bugs.java.com/bugdatabase/view_bug.do?bug_id=6939260 - while ((line = bufferedReader.readLine()) != null) { - processed = processJsonLine(line, batchHandle, - templateMetricVals) || processed; - } - return processed; - } catch (JsonPathNotFoundException | JsonProcessingException e) { - LOG.warn(String.format("Fail to get last modified time of %s ExceptionCode: %s", - file.getAbsolutePath(), StatExceptionCode.JSON_PARSER_ERROR.toString()), e); - StatsCollector.instance().logException(StatExceptionCode.JSON_PARSER_ERROR); - return false; - } - } - - private boolean processJsonLine(String line, BatchBindStep batchHandle, - Object [] templateMetricVals) { - Map map = - JsonConverter.createMapFrom(line); - - if (map.isEmpty()) { - return false; - } - - Object[] metricVals = templateMetricVals.clone(); - int startIndex = derivedDimension.length; - - for (int i = 0; i < directDimensions.length; i++) { - metricVals[startIndex + i] = map.get( - directDimensions[i].toString()); - } - - startIndex += directDimensions.length; - for (int i = 0; i < metadata.length; i++) { - String key = metadata[i].toString(); - if (map.containsKey(key)) { - metricVals[startIndex + i] = map.get(key); - } - } - - batchHandle.bind(metricVals); - return true; - } - - /** - * - * @param snap memory database table representation of metric - * @param startTime when reader starts collecting - * @param lastSnapTimestamp the highest modified time of all the files - * processed for the last snapshot. - * @return whether any metrics extracted from /dev/shm/performanceanalyzer files - * @throws Exception thrown if we have issues parsing metrics - */ - public boolean dispatch(MemoryDBSnapshot snap, - long startTime, long lastSnapTimestamp) throws Exception { - - long startTimeThirtySecondBucket = PerformanceAnalyzerMetrics - .getTimeInterval(startTime); - long prevThirtySecondBucket = startTimeThirtySecondBucket - - MetricsConfiguration.ROTATION_INTERVAL; - - BatchBindStep handle = snap.startBatchPut(); - - boolean metricProcessed = false; - - // TODO: We can have two rows in db tables with the same dimensions. - List metricFiles = handler - .findFiles4Metric(startTimeThirtySecondBucket); - for (File f : metricFiles) { - metricProcessed = processMetrics(f, snap, startTime, - lastSnapTimestamp, handle) || metricProcessed; - } - - metricFiles = handler.findFiles4Metric(prevThirtySecondBucket); - for (File f : metricFiles) { - metricProcessed = processMetrics(f, snap, startTime, - lastSnapTimestamp, handle) || metricProcessed; - } - - if (handle.size() > 0) { - handle.execute(); - } - - return metricProcessed; - } - - private List createEnumNameList(Object[] enumValues) { - if (enumValues != null && enumValues.length > 0) { - return Arrays.stream(enumValues).map( - d -> d.toString()).collect(Collectors.toList()); - } - return Collections.emptyList(); - } - - /** - * Initialize fields used for database operation - */ - private void initializeFields() { - dimensionNames = new ArrayList<>(); - - dimensionNames.addAll(createEnumNameList(derivedDimension)); - - dimensionNames.addAll(createEnumNameList(directDimensions)); - - dimensionFields = DBUtils - .getStringFieldsFromList(dimensionNames); - - metadataTableNames = createEnumNameList(metadata); - metadataFields = DBUtils - .getDoubleFieldsFromList(metadataTableNames); - } - - /** - * Initialize fields used for database operation. Customize some of the - * table names. See the comments of metricTableNames for details. - */ - private void customizeMetricTableName(Map tableName) { - for (int i = 0; i < metadataTableNames.size(); i++) { - String metricName = metadataTableNames.get(i).toString(); - if (tableName.containsKey(metricName)) { - metadataTableNames.set(i, tableName.get(metricName)); - } - } - } - - public List getMetadataTableNames() { - return metadataTableNames; - } - - /** - * Precondition: should be called after metricFields and metricTableNames - * are fully initialized. - * - * Initialize the map from metric table name to select and group by fields. - * These select fields are used to create disk metric table. - */ - private void initializeTableSelectFields() { - - for (int i = 0; i < metadataFields.size(); i++) { - Field metadataField = metadataFields.get(i); - String metadataName = metadataField.getName(); - String tableName = metadataTableNames.get(i); - - List> groupByFields = new ArrayList>(); - groupByFields.addAll(getDimensionFields()); - - List> selectFields = new ArrayList>(); - selectFields.addAll(getDimensionFields()); - - selectFields.add(DSL.sum(metadataField).as("sum_" + metadataName)); - selectFields.add(DSL.avg(metadataField).as("avg_" + metadataName)); - selectFields.add(DSL.min(metadataField).as("min_" + metadataName)); - selectFields.add(DSL.max(metadataField).as("max_" + metadataName)); - - inMemoryTableSelectFieldsMap.put(tableName, selectFields); - inMemoryTableGroupByFieldsMap.put(tableName, groupByFields); - - Condition whereClause = metadataField.isNotNull(); - inMemoryTableWhereClauseMap.put(tableName, whereClause); - } - } - - public Map>> getTableSelectMap() { - return inMemoryTableSelectFieldsMap; - } - - public Map>> getTableGroupByFieldsMap() { - return inMemoryTableGroupByFieldsMap; - } - - public Map getTableWhereClauseMap() { - return inMemoryTableWhereClauseMap; - } - - public List getDimensionNames() { - return dimensionNames; - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesConfig.java deleted file mode 100644 index b1b3f4d1..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesConfig.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import java.io.File; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CircuitBreakerDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CircuitBreakerValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.DiskDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.DiskValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HeapDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HeapValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricName; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardStatsDerivedDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardStatsValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.TCPDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.TCPValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ThreadPoolDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ThreadPoolValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.google.common.annotations.VisibleForTesting; - -public final class MetricPropertiesConfig { - - /** - * Find files under /dev/shm/performanceanalyzer/TS_BUCKET/metricPathElements - * @param metricPathElements path element array - * @return a list of Files - */ - static FileHandler createFileHandler(String... metricPathElements) { - return new FileHandler() { - @Override - public List findFiles4Metric( - long startTimeThirtySecondBucket) { - List ret = new ArrayList(1); - StringBuilder sb = new StringBuilder(); - sb.append(getRootLocation()); - sb.append(startTimeThirtySecondBucket); - - for (String element: metricPathElements) { - sb.append(File.separator); - sb.append(element); - } - File metricFile = new File(sb.toString()); - if (metricFile.exists()) { - ret.add(metricFile); - } - return ret; - } - - }; - } - - public static class ShardStatFileHandler extends FileHandler { - @Override - public List findFiles4Metric(long timeBucket) { - File indicesFolder = new File( - this.getRootLocation() - + File.separator - + timeBucket - + File.separator - + PerformanceAnalyzerMetrics.sIndicesPath); - - if (!indicesFolder.exists()) { - return Collections.emptyList(); - } - - List metricFiles = new ArrayList<>(); - - for (File indexFolder : indicesFolder.listFiles()) { - for (File shardIdFile: indexFolder.listFiles()) { - metricFiles.add(shardIdFile); - } - } - return metricFiles; - } - - @Override - public String filePathRegex() { - // getRootLocation() may or may not end with File.separator. So - // I put ? next to File.separator. - return getRootLocation() + File.separator + "?\\d+" + File.separator - + PerformanceAnalyzerMetrics.sIndicesPath + File.separator + "(.*)" + File.separator - + "(\\d+)"; - } - } - - private final - Map metricName2Property; - - private static final MetricPropertiesConfig INSTANCE = new - MetricPropertiesConfig(); - - private MetricPropertiesConfig() { - metricName2Property = new HashMap<>(); - - metricName2Property.put(MetricName.CIRCUIT_BREAKER, - new MetricProperties(CircuitBreakerDimension.values(), - CircuitBreakerValue.values(), - createFileHandler(PerformanceAnalyzerMetrics.sCircuitBreakerPath))); - metricName2Property.put(MetricName.HEAP_METRICS, - new MetricProperties(HeapDimension.values(), - HeapValue.values(), - createFileHandler(PerformanceAnalyzerMetrics.sHeapPath))); - metricName2Property.put(MetricName.DISK_METRICS, - new MetricProperties(DiskDimension.values(), - DiskValue.values(), - createFileHandler(PerformanceAnalyzerMetrics.sDisksPath))); - metricName2Property.put(MetricName.TCP_METRICS, - new MetricProperties(TCPDimension.values(), - TCPValue.values(), - createFileHandler(PerformanceAnalyzerMetrics.sTCPPath))); - metricName2Property.put(MetricName.IP_METRICS, - new MetricProperties(IPDimension.values(), - IPValue.values(), - createFileHandler(PerformanceAnalyzerMetrics.sIPPath))); - metricName2Property.put(MetricName.THREAD_POOL, - new MetricProperties(ThreadPoolDimension.values(), - ThreadPoolValue.values(), - createFileHandler(PerformanceAnalyzerMetrics.sThreadPoolPath))); - metricName2Property.put(MetricName.SHARD_STATS, - new MetricProperties( - ShardStatsDerivedDimension.values(), - MetricProperties.EMPTY_DIMENSION, - ShardStatsValue.values(), - new ShardStatFileHandler() - )); - metricName2Property.put(MetricName.MASTER_PENDING, - new MetricProperties(MetricProperties.EMPTY_DIMENSION, - MasterPendingValue.values(), - createFileHandler(PerformanceAnalyzerMetrics.sPendingTasksPath, - PerformanceAnalyzerMetrics.MASTER_CURRENT, - PerformanceAnalyzerMetrics.MASTER_META_DATA) - )); - } - - public static MetricPropertiesConfig getInstance() { - return INSTANCE; - } - - public MetricProperties getProperty(MetricName name) { - return metricName2Property.get(name); - } - - @VisibleForTesting - Map getMetricName2Property() { - return metricName2Property; - } - -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricsEmitter.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricsEmitter.java deleted file mode 100644 index 408442ac..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricsEmitter.java +++ /dev/null @@ -1,608 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jooq.BatchBindStep; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.SelectField; -import org.jooq.SelectHavingStep; -import org.jooq.impl.DSL; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.DBUtils; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.TroubleshootingConfig; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CommonMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.OSMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.Dimensions; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.Metric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; - -@SuppressWarnings("serial") -public class MetricsEmitter { - - private static final Logger LOG = LogManager.getLogger(MetricsEmitter.class); - - private static final Pattern GC_PATTERN = Pattern.compile(".*(GC|CMS|Parallel).*"); - private static final Pattern REFRESH_PATTERN = Pattern.compile(".*elasticsearch.*\\[refresh\\].*"); - private static final Pattern MANAGEMENT_PATTERN = Pattern.compile(".*elasticsearch.*\\[management\\].*"); - private static final Pattern MERGE_PATTERN = Pattern.compile(".*elasticsearch\\[.*\\]\\[\\[(.*)\\]\\[(.*)\\].*Lucene Merge.*"); - private static final Pattern SEARCH_PATTERN = Pattern.compile(".*elasticsearch.*\\[search\\].*"); - private static final Pattern BULK_PATTERN = Pattern.compile(".*elasticsearch.*\\[bulk\\].*"); - //ES 6.4 onwards uses write threadpool. - private static final Pattern WRITE_PATTERN = Pattern.compile(".*elasticsearch.*\\[write\\].*"); - //Pattern otherPattern = Pattern.compile(".*(elasticsearch).*"); - private static final Pattern HTTP_SERVER_PATTERN = Pattern.compile(".*elasticsearch.*\\[http_server_worker\\].*"); - private static final Pattern TRANS_SERVER_PATTERN = Pattern.compile(".*elasticsearch.*\\[transport_server_worker.*"); - private static final Pattern TRANS_CLIENT_PATTERN = Pattern.compile(".*elasticsearch.*\\[transport_client_boss\\].*"); - - private static final List LATENCY_TABLE_DIMENSIONS = new ArrayList() { { - this.add(ShardRequestMetricsSnapshot.Fields.OPERATION.toString()); - this.add(HttpRequestMetricsSnapshot.Fields.EXCEPTION.toString()); - this.add(HttpRequestMetricsSnapshot.Fields.INDICES.toString()); - this.add(HttpRequestMetricsSnapshot.Fields.HTTP_RESP_CODE.toString()); - this.add(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString()); - this.add(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString()); - this.add(ShardRequestMetricsSnapshot.Fields.SHARD_ROLE.toString()); - } }; - - - public static void emitAggregatedOSMetrics(final DSLContext create, - final MetricsDB db, final OSMetricsSnapshot osMetricsSnap, - final ShardRequestMetricsSnapshot rqMetricsSnap) throws Exception { - - SelectHavingStep rqTable = rqMetricsSnap.fetchThreadUtilizationRatioTable(); - SelectHavingStep osTable = osMetricsSnap.selectAll(); - - List> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString()), String.class)); - this.add(DSL.field(DSL.name(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString()), String.class)); - this.add(DSL.field(DSL.name(ShardRequestMetricsSnapshot.Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(ShardRequestMetricsSnapshot.Fields.SHARD_ROLE.toString()), String.class)); - } }; - - for (OSMetrics metric: OSMetrics.values()) { - fields.add(DSL.field(ShardRequestMetricsSnapshot.Fields.TUTIL.toString(), Double.class).mul( - DSL.field(DSL.name(metric.toString()), Double.class)) - .as(metric.toString())); - } - - ArrayList> groupByFields = new ArrayList>() { { - this.add(DSL.field(DSL.name(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString()), String.class)); - this.add(DSL.field(DSL.name(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString()), String.class)); - this.add(DSL.field(DSL.name(ShardRequestMetricsSnapshot.Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(ShardRequestMetricsSnapshot.Fields.SHARD_ROLE.toString()), String.class)); - } }; - - List> aggFields = new ArrayList>() { { - this.add(DSL.field(DSL.name(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString()), String.class)); - this.add(DSL.field(DSL.name(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString()), String.class)); - this.add(DSL.field(DSL.name(ShardRequestMetricsSnapshot.Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(ShardRequestMetricsSnapshot.Fields.SHARD_ROLE.toString()), String.class)); - } }; - - for (OSMetrics metric: OSMetrics.values()) { - aggFields.add(DSL.sum(DSL.field(DSL.name(metric.toString()), Double.class)) - .as(MetricsDB.SUM + "_" + metric.toString())); - aggFields.add(DSL.avg(DSL.field(DSL.name(metric.toString()), Double.class)) - .as(MetricsDB.AVG + "_" + metric.toString())); - aggFields.add(DSL.min(DSL.field(DSL.name(metric.toString()), Double.class)) - .as(MetricsDB.MIN + "_" + metric.toString())); - aggFields.add(DSL.max(DSL.field(DSL.name(metric.toString()), Double.class)) - .as(MetricsDB.MAX + "_" + metric.toString())); - } - - long mCurrT = System.currentTimeMillis(); - Result res = create.select(aggFields) - .from( - create.select(fields) - .from(rqTable) - .join(osTable) - .on(osTable.field(OSMetricsSnapshot.Fields.tid.toString(), String.class).eq( - rqTable.field(OSMetricsSnapshot.Fields.tid.toString(), String.class))) - ) - .groupBy(groupByFields) - .fetch(); - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for tid corelation: {}", mFinalT - mCurrT); - checkInvalidData(rqTable, osTable, create); - - Set metricColumns = osMetricsSnap.getMetricColumns(); - - mCurrT = System.currentTimeMillis(); - for (String metricColumn: metricColumns) { - List dims = new ArrayList() { { - this.add(AllMetrics.CommonDimension.SHARD_ID.toString()); - this.add(AllMetrics.CommonDimension.INDEX_NAME.toString()); - this.add(AllMetrics.CommonDimension.OPERATION.toString()); - this.add(AllMetrics.CommonDimension.SHARD_ROLE.toString()); - } }; - db.createMetric(new Metric(metricColumn, 0d), - dims); - BatchBindStep handle = db.startBatchPut(new Metric(metricColumn, 0d), - dims); - for (Record r: res) { - if (r.get(MetricsDB.SUM + "_" + metricColumn) == null) { - continue; - } - - Double sumMetric = Double.parseDouble(r.get(MetricsDB.SUM + "_" + metricColumn).toString()); - Double avgMetric = Double.parseDouble(r.get(MetricsDB.AVG + "_" + metricColumn).toString()); - Double minMetric = Double.parseDouble(r.get(MetricsDB.MIN + "_" + metricColumn).toString()); - Double maxMetric = Double.parseDouble(r.get(MetricsDB.MAX + "_" + metricColumn).toString()); - handle.bind(r.get(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString()).toString(), - r.get(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString()).toString(), - r.get(ShardRequestMetricsSnapshot.Fields.OPERATION.toString()).toString(), - r.get(ShardRequestMetricsSnapshot.Fields.SHARD_ROLE.toString()).toString(), - sumMetric, avgMetric, minMetric, maxMetric); - } - - if (handle.size() > 0) { - handle.execute(); - } - } - mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for writing resource metrics metricsdb: {}", mFinalT - mCurrT); - } - - /** - * Check if there is any invalid data. - * Invalid data is if we have tid in request table but not in OS tables. - * @param rqTable request table select - * @param osTable OS table select - * @param create db connection - */ - @SuppressWarnings("unchecked") - private static void checkInvalidData(SelectHavingStep rqTable, SelectHavingStep osTable, - final DSLContext create) { - if (!TroubleshootingConfig.getEnableDevAssert()) { - return; - } - - Condition condition = DSL.trueCondition(); - Field tidField = DSL.field(DSL.name(OSMetricsSnapshot.Fields.tid.toString()), String.class); - Field tNameField = DSL.field(DSL.name(OSMetricsSnapshot.Fields.tName.toString()), String.class); - - Set rqSet = DBUtils.getRecordSetByField(rqTable, tidField, condition, create); - condition = tNameField.contains("[bulk]").or(tNameField.contains("[search]")); - Set osSet = DBUtils.getRecordSetByField(osTable, tidField, condition, create); - - if (!osSet.containsAll(rqSet)) { - String msg = String.format("[Invalid Data] Unmatched tid between %s and %s", rqSet.toString(), osSet.toString()); - LOG.error(msg); - LOG.error(create.select().from(rqTable).fetch().toString()); - LOG.error(create.select().from(osTable).where(condition).fetch().toString()); - throw new RuntimeException(msg); - } - } - - public static void emitWorkloadMetrics(final DSLContext create, final MetricsDB db, - final ShardRequestMetricsSnapshot rqMetricsSnap) throws Exception { - long mCurrT = System.currentTimeMillis(); - Result res = rqMetricsSnap.fetchLatencyByOp(); - - db.createMetric(new Metric(CommonMetric.LATENCY.toString(), 0d), - LATENCY_TABLE_DIMENSIONS); - BatchBindStep handle = db.startBatchPut(new Metric( - CommonMetric.LATENCY.toString(), 0d), LATENCY_TABLE_DIMENSIONS); - - //Dims need to be changed. - List shardDims = new ArrayList() { { - this.add(ShardRequestMetricsSnapshot.Fields.OPERATION.toString()); - this.add(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString()); - this.add(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString()); - this.add(ShardRequestMetricsSnapshot.Fields.SHARD_ROLE.toString()); - } }; - - db.createMetric(new Metric(AllMetrics.ShardOperationMetric.SHARD_OP_COUNT.toString(), 0d), - shardDims); - BatchBindStep countHandle = db.startBatchPut(new Metric( - AllMetrics.ShardOperationMetric.SHARD_OP_COUNT.toString(), 0d), shardDims); - - db.createMetric(new Metric(AllMetrics.ShardBulkMetric.DOC_COUNT.toString(), 0d), - shardDims); - BatchBindStep bulkDocHandle = db.startBatchPut(new Metric( - AllMetrics.ShardBulkMetric.DOC_COUNT.toString(), 0d), shardDims); - - for (Record r: res) { - Double sumLatency = Double.parseDouble(r.get(DBUtils. - getAggFieldName(ShardRequestMetricsSnapshot.Fields.LAT.toString(), MetricsDB.SUM)) - .toString()); - Double avgLatency = Double.parseDouble(r.get(DBUtils. - getAggFieldName(ShardRequestMetricsSnapshot.Fields.LAT.toString(), MetricsDB.AVG)) - .toString()); - Double minLatency = Double.parseDouble(r.get(DBUtils. - getAggFieldName(ShardRequestMetricsSnapshot.Fields.LAT.toString(), MetricsDB.MIN)) - .toString()); - Double maxLatency = Double.parseDouble(r.get(DBUtils. - getAggFieldName(ShardRequestMetricsSnapshot.Fields.LAT.toString(), MetricsDB.MAX)) - .toString()); - - handle.bind(r.get(ShardRequestMetricsSnapshot.Fields.OPERATION.toString()).toString(), - null, - null, - null, - r.get(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString()).toString(), - r.get(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString()).toString(), - r.get(ShardRequestMetricsSnapshot.Fields.SHARD_ROLE.toString()).toString(), - sumLatency, - avgLatency, - minLatency, - maxLatency - ); - - Double count = Double.parseDouble(r.get(AllMetrics.ShardOperationMetric.SHARD_OP_COUNT.toString()).toString()); - countHandle.bind(r.get(ShardRequestMetricsSnapshot.Fields.OPERATION.toString()).toString(), - r.get(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString()).toString(), - r.get(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString()).toString(), - r.get(ShardRequestMetricsSnapshot.Fields.SHARD_ROLE.toString()).toString(), - count, - count, - count, - count - ); - - Object bulkDocCountObj = r.get(AllMetrics.ShardBulkMetric.DOC_COUNT.toString()); - if (bulkDocCountObj != null) { - Double bulkDocCount = Double.parseDouble(bulkDocCountObj.toString()); - bulkDocHandle.bind(r.get(ShardRequestMetricsSnapshot.Fields.OPERATION.toString()).toString(), - r.get(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString()).toString(), - r.get(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString()).toString(), - r.get(ShardRequestMetricsSnapshot.Fields.SHARD_ROLE.toString()).toString(), - bulkDocCount, - bulkDocCount, - bulkDocCount, - bulkDocCount - ); - } - } - if (handle.size() > 0) { - handle.execute(); - } - if (countHandle.size() > 0) { - countHandle.execute(); - } - if (bulkDocHandle.size() > 0) { - bulkDocHandle.execute(); - } - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for writing workload metrics metricsdb: {}", mFinalT - mCurrT); - } - - public static void emitThreadNameMetrics(final DSLContext create, final MetricsDB db, - final OSMetricsSnapshot osMetricsSnap) throws Exception { - long mCurrT = System.currentTimeMillis(); - Result res = osMetricsSnap.getOSMetrics(); - - Set metricColumns = osMetricsSnap.getMetricColumns(); - for (Record r: res) { - Dimensions dimensions = new Dimensions(); - Object threadName = r.get(OSMetricsSnapshot.Fields.tName.toString()); - - if (threadName == null) { - LOG.debug("Could not find tName: {}", r); - continue; - } - String operation = categorizeThreadName(threadName.toString(), dimensions); - if (operation == null) { - continue; - } - - dimensions.put(ShardRequestMetricsSnapshot.Fields.OPERATION.toString(), operation); - for (String metricColumn: metricColumns) { - if (r.get(metricColumn) == null) { - continue; - } - Double metric = Double.parseDouble(r.get(metricColumn).toString()); - if (operation.equals("merge") && metricColumn.equals("cpu")) { - LOG.debug("Putting merge metric {}", metric); - } - db.putMetric(new Metric(metricColumn, metric), - dimensions, 0); - } - } - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for writing threadName metrics metricsdb: {}", mFinalT - mCurrT); - } - - public static String categorizeThreadName(String threadName, Dimensions dimensions) { - //shardSearch and shardBulk os metrics are emitted by emitAggregatedOSMetrics and emitWorkloadMetrics functions. - //Hence these are ignored in this emitter. - if (SEARCH_PATTERN.matcher(threadName).matches()) { - return null; - } - if (BULK_PATTERN.matcher(threadName).matches() || WRITE_PATTERN.matcher(threadName).matches()) { - return null; - } - - if (GC_PATTERN.matcher(threadName).matches()) { - return "GC"; - } - if (REFRESH_PATTERN.matcher(threadName).matches()) { - return "refresh"; - } - if (MANAGEMENT_PATTERN.matcher(threadName).matches()) { - return "management"; - } - if (HTTP_SERVER_PATTERN.matcher(threadName).matches()) { - return "httpServer"; - } - if (TRANS_CLIENT_PATTERN.matcher(threadName).matches()) { - return "transportClient"; - } - if (TRANS_SERVER_PATTERN.matcher(threadName).matches()) { - return "transportServer"; - } - Matcher mergeMatcher = MERGE_PATTERN.matcher(threadName); - if (mergeMatcher.matches()) { - dimensions.put( - ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString(), - mergeMatcher.group(1)); - dimensions.put( - ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString(), - mergeMatcher.group(2)); - return "merge"; - } - return "other"; - } - - public static void emitHttpMetrics(final DSLContext create, final MetricsDB db, - final HttpRequestMetricsSnapshot rqMetricsSnap) throws Exception { - long mCurrT = System.currentTimeMillis(); - Dimensions dimensions = new Dimensions(); - Result res = rqMetricsSnap.fetchLatencyByOp(); - List dims = new ArrayList() { { - this.add(HttpRequestMetricsSnapshot.Fields.OPERATION.toString()); - this.add(HttpRequestMetricsSnapshot.Fields.EXCEPTION.toString()); - this.add(HttpRequestMetricsSnapshot.Fields.INDICES.toString()); - this.add(HttpRequestMetricsSnapshot.Fields.HTTP_RESP_CODE.toString()); - } }; - - db.createMetric(new Metric(AllMetrics.CommonMetric.LATENCY.toString(), 0d), - LATENCY_TABLE_DIMENSIONS); - - db.createMetric(new Metric(AllMetrics.HttpMetric.HTTP_TOTAL_REQUESTS.toString(), 0d), - dims); - db.createMetric(new Metric(AllMetrics.HttpMetric.HTTP_REQUEST_DOCS.toString(), 0d), - dims); - - for (Record r: res) { - dimensions.put(HttpRequestMetricsSnapshot.Fields.OPERATION.toString(), - r.get(HttpRequestMetricsSnapshot.Fields.OPERATION.toString()).toString()); - dimensions.put(HttpRequestMetricsSnapshot.Fields.HTTP_RESP_CODE.toString(), - r.get(HttpRequestMetricsSnapshot.Fields.HTTP_RESP_CODE.toString()).toString()); - dimensions.put(HttpRequestMetricsSnapshot.Fields.INDICES.toString(), - r.get(HttpRequestMetricsSnapshot.Fields.INDICES.toString()).toString()); - dimensions.put(HttpRequestMetricsSnapshot.Fields.EXCEPTION.toString(), - r.get(HttpRequestMetricsSnapshot.Fields.EXCEPTION.toString()).toString()); - - Double sumLatency = Double.parseDouble(r.get(DBUtils. - getAggFieldName(HttpRequestMetricsSnapshot.Fields.LAT.toString(), MetricsDB.SUM)) - .toString()); - Double avgLatency = Double.parseDouble(r.get(DBUtils. - getAggFieldName(HttpRequestMetricsSnapshot.Fields.LAT.toString(), MetricsDB.AVG)) - .toString()); - Double minLatency = Double.parseDouble(r.get(DBUtils. - getAggFieldName(HttpRequestMetricsSnapshot.Fields.LAT.toString(), MetricsDB.MIN)) - .toString()); - Double maxLatency = Double.parseDouble(r.get(DBUtils. - getAggFieldName(HttpRequestMetricsSnapshot.Fields.LAT.toString(), MetricsDB.MAX)) - .toString()); - - Double count = Double.parseDouble(r.get(HttpRequestMetricsSnapshot.Fields.HTTP_TOTAL_REQUESTS.toString()).toString()); - - Double sumItemCount = Double.parseDouble(r.get(DBUtils. - getAggFieldName(HttpRequestMetricsSnapshot.Fields.HTTP_REQUEST_DOCS.toString(), MetricsDB.SUM)) - .toString()); - Double avgItemCount = Double.parseDouble(r.get(DBUtils. - getAggFieldName(HttpRequestMetricsSnapshot.Fields.HTTP_REQUEST_DOCS.toString(), MetricsDB.AVG)) - .toString()); - Double minItemCount = Double.parseDouble(r.get(DBUtils. - getAggFieldName(HttpRequestMetricsSnapshot.Fields.HTTP_REQUEST_DOCS.toString(), MetricsDB.MIN)) - .toString()); - Double maxItemCount = Double.parseDouble(r.get(DBUtils. - getAggFieldName(HttpRequestMetricsSnapshot.Fields.HTTP_REQUEST_DOCS.toString(), MetricsDB.MAX)) - .toString()); - - db.putMetric(new Metric(AllMetrics.CommonMetric.LATENCY.toString(), sumLatency, - avgLatency, minLatency, maxLatency), - dimensions, 0); - db.putMetric(new Metric(AllMetrics.HttpMetric.HTTP_TOTAL_REQUESTS.toString(), count), - dimensions, 0); - db.putMetric(new Metric(AllMetrics.HttpMetric.HTTP_REQUEST_DOCS.toString(), sumItemCount, - avgItemCount, minItemCount, maxItemCount), - dimensions, 0); - } - - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for writing http metrics metricsdb: {}", mFinalT - mCurrT); - } - - public static void emitMasterEventMetrics(MetricsDB metricsDB, MasterEventMetricsSnapshot masterEventMetricsSnapshot) { - - long mCurrT = System.currentTimeMillis(); - Result queueAndRunTimeResult = masterEventMetricsSnapshot.fetchQueueAndRunTime(); - - List dims = new ArrayList() { { - this.add(AllMetrics.MasterMetricDimensions.MASTER_TASK_INSERT_ORDER.toString()); - this.add(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString()); - this.add(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString()); - this.add(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString()); - } }; - - emitQueueTimeMetric(metricsDB, queueAndRunTimeResult, dims); - emitRuntimeMetric(metricsDB, queueAndRunTimeResult, dims); - - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for writing master event queue metrics metricsdb: {}", mFinalT - mCurrT); - } - - private static void emitRuntimeMetric(MetricsDB metricsDB, Result res, List dims) { - - metricsDB.createMetric( - new Metric(AllMetrics.MasterMetricValues.MASTER_TASK_RUN_TIME.toString(), 0d), dims); - - BatchBindStep handle = metricsDB.startBatchPut(new Metric( - AllMetrics.MasterMetricValues.MASTER_TASK_RUN_TIME.toString(), 0d), dims); - - for (Record r: res) { - - Double sumQueueTime = Double.parseDouble(r.get(DBUtils. - getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString(), MetricsDB.SUM)) - .toString()); - - Double avgQueueTime = Double.parseDouble(r.get(DBUtils. - getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString(), MetricsDB.AVG)) - .toString()); - - Double minQueueTime = Double.parseDouble(r.get(DBUtils. - getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString(), MetricsDB.MIN)) - .toString()); - - Double maxQueueTime = Double.parseDouble(r.get(DBUtils. - getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString(), MetricsDB.MAX)) - .toString()); - - handle.bind(r.get(AllMetrics.MasterMetricDimensions.MASTER_TASK_INSERT_ORDER.toString()).toString(), - r.get(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString()).toString(), - r.get(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString()).toString(), - r.get(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString()).toString(), - sumQueueTime, - avgQueueTime, - minQueueTime, - maxQueueTime); - } - - handle.execute(); - } - - private static void emitQueueTimeMetric(MetricsDB metricsDB, Result res, List dims) { - - metricsDB.createMetric( - new Metric(AllMetrics.MasterMetricValues.MASTER_TASK_QUEUE_TIME.toString(), 0d), dims); - - BatchBindStep handle = metricsDB.startBatchPut(new Metric( - AllMetrics.MasterMetricValues.MASTER_TASK_QUEUE_TIME.toString(), 0d), dims); - - for (Record r: res) { - - Double sumQueueTime = Double.parseDouble(r.get(DBUtils. - getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString(), MetricsDB.SUM)) - .toString()); - - Double avgQueueTime = Double.parseDouble(r.get(DBUtils. - getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString(), MetricsDB.AVG)) - .toString()); - - Double minQueueTime = Double.parseDouble(r.get(DBUtils. - getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString(), MetricsDB.MIN)) - .toString()); - - Double maxQueueTime = Double.parseDouble(r.get(DBUtils. - getAggFieldName(AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString(), MetricsDB.MAX)) - .toString()); - - handle.bind(r.get(AllMetrics.MasterMetricDimensions.MASTER_TASK_INSERT_ORDER.toString()).toString(), - r.get(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString()).toString(), - r.get(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString()).toString(), - r.get(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString()).toString(), - sumQueueTime, - avgQueueTime, - minQueueTime, - maxQueueTime); - } - - handle.execute(); - } - - /** - * TODO: Some of these metrics have default value like tcp.SSThresh:-1. - * Should we count them in aggregation? - * @param create A contextual DSL providing "attached" implementations to - * the org.jooq interfaces. - * @param db On-disk database that holds a snapshot of all metrics, which - * includes the metrics that customers can query. - *@param snap In memory database that holds a snapshot of all metrics. - * This is the intermediate representation of metrics. - * @throws Exception thrown when we cannot emit metrics from the in-memory - * database to the on-disk database. - */ - public static void emitNodeMetrics(final DSLContext create, - final MetricsDB db, final MemoryDBSnapshot snap) throws Exception { - - Map> metadataTable = snap - .selectMetadataSource(); - - Map>> selectField = snap - .getTableSelectFieldsMap(); - - List dimensionNames = snap.getDimensionNames(); - - for (Map.Entry> entry : metadataTable - .entrySet()) { - long mCurrT = System.currentTimeMillis(); - - String tableName = entry.getKey(); - Result fetchedData = entry.getValue().fetch(); - - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for aggregating {} : {}", tableName, - mFinalT - mCurrT); - - if (fetchedData == null || fetchedData.size() == 0) { - LOG.info("No data to emit: {}", tableName); - continue; - } - - mCurrT = System.currentTimeMillis(); - - List> selectFields = selectField.get(tableName); - - db.createMetric(new Metric(tableName, 0d), - dimensionNames); - - BatchBindStep handle = db.startBatchPut(tableName, - selectFields.size()); - for (Record r : fetchedData) { - int columnNum = selectFields.size(); - Object[] bindValues = new Object[columnNum]; - for (int i = 0; i < columnNum; i++) { - bindValues[i] = r.get(selectFields.get(i).getName()); - } - handle.bind(bindValues); - } - handle.execute(); - - mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for writing {} metrics metricsdb: {}", - tableName, mFinalT - mCurrT); - } - } -} - - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricsParser.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricsParser.java deleted file mode 100644 index a3911469..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricsParser.java +++ /dev/null @@ -1,496 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import java.io.File; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jooq.BatchBindStep; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.OSMetricsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HttpDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HttpMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.OSMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardBulkDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardBulkMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CommonMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterMetricDimensions; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.FileHelper; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; - -/** - * Read metrics files emitted by Elasticsearch in /dev/shm and efficiently load them into tables for further processing. - */ -public class MetricsParser { - private static final Logger LOG = LogManager.getLogger(MetricsParser.class); - - public long getThirtySecondBucket(long startTime) { - return PerformanceAnalyzerMetrics.getTimeInterval(startTime); - } - - public void parseOSMetrics(String rootLocation, long startTime, long endTime, - OSMetricsSnapshot osMetricsSnap) throws Exception { - long startTimeThirtySecondBucket = getThirtySecondBucket(startTime); - - File threadsFile = new File(rootLocation + File.separator - + startTimeThirtySecondBucket + File.separator - + PerformanceAnalyzerMetrics.sThreadsPath); - - BatchBindStep handle = osMetricsSnap.startBatchPut(); - List tidToDelete = new ArrayList<>(); - processOSMetricsForFile(threadsFile, osMetricsSnap, startTime, endTime, handle, tidToDelete); - - osMetricsSnap.deleteByTid(tidToDelete); - - if (handle.size() > 0) { - handle.execute(); - } - } - - public void processOSMetricsForFile(File threadsFile, - OSMetricsSnapshot osMetricsSnap, long startTime, long endTime, - BatchBindStep batchHandle, List tidToDelete) throws Exception { - - Map lastUpdateTimePerTid = osMetricsSnap.getLastUpdateTimePerTid(); - - boolean retVal = false; - if (threadsFile.exists()) { - for (File threadIDFile: threadsFile.listFiles()) { - if (!threadIDFile.getName().equals(PerformanceAnalyzerMetrics.sHttpPath)) { - String threadID = threadIDFile.getName(); - - for (File opFile: threadIDFile.listFiles()) { - if (opFile.getName().equals(PerformanceAnalyzerMetrics.sOSPath)) { - retVal = processOSMetrics(opFile, threadID, lastUpdateTimePerTid, startTime, - endTime, batchHandle, tidToDelete) - || retVal; - } - } - } - } - } - LOG.info("processOSMetricsForFile ret: {}", retVal);; - } - - public void parseRequestMetrics(String rootLocation, long startTime, - long endTime, ShardRequestMetricsSnapshot rqMetricsSnap) throws Exception { - - long startTimeThirtySecondBucket = getThirtySecondBucket(startTime); - File threadsFile = new File(rootLocation + File.separator - + startTimeThirtySecondBucket + File.separator - + PerformanceAnalyzerMetrics.sThreadsPath); - - BatchBindStep handle = rqMetricsSnap.startBatchPut(); - if (threadsFile.exists()) { - for (File threadIDFile: threadsFile.listFiles()) { - if (!threadIDFile.getName().equals(PerformanceAnalyzerMetrics.sHttpPath)) { - String threadID = threadIDFile.getName(); - - for (File opFile: threadIDFile.listFiles()) { - if (opFile.getName().equals(PerformanceAnalyzerMetrics.sShardBulkPath) - || opFile.getName().equals(PerformanceAnalyzerMetrics.sShardFetchPath) - || opFile.getName().equals(PerformanceAnalyzerMetrics.sShardQueryPath)) { - handleESMetrics(opFile, threadID, startTime, endTime, handle); - } - } - } - } - } - - if (handle.size() > 0) { - handle.execute(); - } - } - - public void parseHttpMetrics(String rootLocation, long startTime, - long endTime, HttpRequestMetricsSnapshot rqMetricsSnap) throws Exception { - - long startTimeThirtySecondBucket = getThirtySecondBucket(startTime); - File httpFile = new File(rootLocation + File.separator - + startTimeThirtySecondBucket + File.separator - + PerformanceAnalyzerMetrics.sThreadsPath + File.separator + PerformanceAnalyzerMetrics.sHttpPath); - - BatchBindStep handle = rqMetricsSnap.startBatchPut(); - - if (httpFile.exists()) { - for (File opFile: httpFile.listFiles()) { - String operation = opFile.getName(); - for (File rFile: opFile.listFiles()) { - String requestId = rFile.getName(); - for (File metricsFile: rFile.listFiles()) { - long lastModified = FileHelper.getLastModified(metricsFile, startTime, endTime); - if (lastModified < startTime || lastModified >= endTime) { - continue; - } - try { - if (metricsFile.getName().equals(PerformanceAnalyzerMetrics.START_FILE_NAME)) { - emitStartHttpMetric(metricsFile, requestId, operation, handle); - } else if (metricsFile.getName().equals(PerformanceAnalyzerMetrics.FINISH_FILE_NAME)) { - emitFinishHttpMetric(metricsFile, requestId, operation, handle); - } - } catch (Exception e) { - LOG.error(e, e); - LOG.error("Error parsing file - {} ExcepionCode: {}\n", - metricsFile.getAbsolutePath(), StatExceptionCode.READER_PARSER_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.READER_PARSER_ERROR); - throw e; - } - } - } - } - } - - if (handle.size() > 0) { - handle.execute(); - } - } - - public void parseMasterEventMetrics(String rootLocation, - long startTime, - long endTime, - MasterEventMetricsSnapshot masterEventMetricsSnapshot) { - - long startTimeThirtySecondBucket = getThirtySecondBucket(startTime); - File threadsFile = new File(rootLocation + File.separator - + startTimeThirtySecondBucket + File.separator - + PerformanceAnalyzerMetrics.sThreadsPath); - - BatchBindStep handle = masterEventMetricsSnapshot.startBatchPut(); - if (threadsFile.exists()) { - - try { - // Perform level order traversal on file directories - Queue queue = new LinkedList<>(); - Queue idQueue = new LinkedList<>(); - - expandThreadDirectory(threadsFile, queue); - expandThreadIDDirectory(queue, idQueue); - expandOperationDirectory(queue, idQueue); - expandInsertOrderDirectory(queue, idQueue); - - emitMasterStartFinishMetrics(startTime, endTime, handle, queue, idQueue); - } catch (Exception e) { - LOG.error("Failed to parse master metrics with ExceptionCode: " + StatExceptionCode.READER_PARSER_ERROR.toString(), e); - StatsCollector.instance().logException(StatExceptionCode.READER_PARSER_ERROR); - } - } - - if (handle.size() > 0) { - handle.execute(); - } - } - - private void emitMasterStartFinishMetrics(long startTime, - long endTime, - BatchBindStep handle, - Queue queue, - Queue idQueue) { - - // process start and finish - while (!queue.isEmpty()) { - File metricsFile = queue.poll(); - String threadID = idQueue.poll(); - String insertOder = idQueue.poll(); - - long lastModified = FileHelper.getLastModified(metricsFile, startTime, endTime); - if (lastModified < startTime || lastModified >= endTime) { - continue; - } - - String metrics = PerformanceAnalyzerMetrics.getMetric(metricsFile.getAbsolutePath()); - try { - if (metricsFile.getName().equals(PerformanceAnalyzerMetrics.START_FILE_NAME)) { - emitStartMasterEventMetric(metrics, insertOder, threadID, handle); - } else if (metricsFile.getName().equals(PerformanceAnalyzerMetrics.FINISH_FILE_NAME)) { - emitEndMasterEventMetric(metrics, insertOder, threadID, handle); - } - } catch (Exception e) { - LOG.error(e, e); - LOG.error("Error parsing file - {} ExcepionCode: {},\n {}", - metricsFile.getAbsolutePath(), StatExceptionCode.READER_PARSER_ERROR.toString(), metrics); - StatsCollector.instance().logException(StatExceptionCode.READER_PARSER_ERROR); - } - } - } - - private void expandInsertOrderDirectory(Queue queue, Queue idQueue) { - - int size = queue.size(); - for (int i = 0; i < size; i++) { - File insertOrderFile = queue.poll(); - String threadID = idQueue.poll(); - String insertOder = insertOrderFile.getName(); - - for (File metricsFile: insertOrderFile.listFiles()) { - queue.add(metricsFile); - idQueue.add(threadID); - idQueue.add(insertOder); - } - } - } - - private void expandOperationDirectory(Queue queue, Queue idQueue) { - - int size = queue.size(); - for (int i = 0; i < size; i++) { - File opFile = queue.poll(); - String threadId = idQueue.poll(); - - for (File insertOrderFile : opFile.listFiles()) { - queue.add(insertOrderFile); - idQueue.add(threadId); - } - } - } - - private void expandThreadIDDirectory(Queue queue, Queue idQueue) { - - int size = queue.size(); - for (int i = 0; i < size; i++) { - File threadIDFile = queue.poll(); - String threadID = threadIDFile.getName(); - - for (File opFile : threadIDFile.listFiles()) { - if (opFile.getName().equals(PerformanceAnalyzerMetrics.sMasterTaskPath)) { - queue.add(opFile); - idQueue.add(threadID); - } - } - } - } - - private void expandThreadDirectory(File threadsFile, Queue queue) { - - for (File threadIDFile: threadsFile.listFiles()) { - if (!threadIDFile.getName().equals(PerformanceAnalyzerMetrics.sHttpPath)) { - queue.add(threadIDFile); - } - } - } - - private void emitStartMasterEventMetric(String startMetrics, - String insertOrder, - String threadId, - BatchBindStep handle) { - - String priority = PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, - MasterMetricDimensions.MASTER_TASK_PRIORITY.toString()); - - long st = Long.parseLong(PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, - CommonMetric.START_TIME.toString())); - - String taskType = PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, - MasterMetricDimensions.MASTER_TASK_TYPE.toString()); - - String taskMetadata = PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, - MasterMetricDimensions.MASTER_TASK_METADATA.toString()); - - long queueTime = Long.parseLong(PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, - MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString())); - - handle.bind(threadId, insertOrder, priority, taskType, taskMetadata, queueTime, st, null); - } - - private void emitEndMasterEventMetric(String startMetrics, - String insertOrder, - String threadId, - BatchBindStep handle) { - - long finishTime = Long.parseLong(PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, - CommonMetric.FINISH_TIME.toString())); - handle.bind(threadId, insertOrder, null, null, null, null, null, finishTime); - } - - private void emitStartHttpMetric(File metricFile, String rid, - String operation, BatchBindStep handle) { - - String startMetrics = PerformanceAnalyzerMetrics.getMetric(metricFile.getAbsolutePath()); - String startTimeVal = PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, HttpMetric.START_TIME.toString()); - String itemCountVal = PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, HttpMetric.HTTP_REQUEST_DOCS.toString()); - try { - long st = Long.parseLong(startTimeVal); - String indices = PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, HttpDimension.INDICES.toString()); - long itemCount = Long.parseLong(itemCountVal); - handle.bind(rid, operation, indices, null, null, itemCount, st, null); - } catch (NumberFormatException e) { - LOG.error("Unable to parse string. StartTime:{}, itemCount:{}, ExcepionCode: {},\n startMetrics:{}", - startTimeVal, itemCountVal, StatExceptionCode.READER_PARSER_ERROR.toString(), startMetrics); - StatsCollector.instance().logException(StatExceptionCode.READER_PARSER_ERROR); - throw e; - } - } - - private void emitFinishHttpMetric(File metricFile, String rid, - String operation, BatchBindStep handle) { - String finishMetrics = PerformanceAnalyzerMetrics.getMetric(metricFile.getAbsolutePath()); - - - String finishTimeVal = PerformanceAnalyzerMetrics.extractMetricValue(finishMetrics, HttpMetric.FINISH_TIME.toString()); - String status = PerformanceAnalyzerMetrics.extractMetricValue(finishMetrics, HttpDimension.HTTP_RESP_CODE.toString()); - String exception = PerformanceAnalyzerMetrics.extractMetricValue(finishMetrics, HttpDimension.EXCEPTION.toString()); - try { - long ft = Long.parseLong(finishTimeVal); - handle.bind(rid, operation, null, status, exception, null, null, ft); - } catch (NumberFormatException e) { - LOG.error("Unable to parse string. FinishTime:{} ExcepionCode: {} \n finishMetrics:{}", - finishTimeVal, StatExceptionCode.READER_PARSER_ERROR.toString(), finishMetrics); - StatsCollector.instance().logException(StatExceptionCode.READER_PARSER_ERROR); - throw e; - } - } - - private boolean processOSMetrics(File opFile, String threadID, - Map lastUpdateTimePerTid, - long startTime, long endTime, - BatchBindStep batchHandle, List tidToDelete) throws Exception { - Map osMetrics = new HashMap<>(); - long opFileLastModified = FileHelper.getLastModified(opFile, startTime, endTime); - if (opFileLastModified > endTime) { - opFileLastModified = endTime; - LOG.info("File last modified time is greater than endTime - {}", opFile.getAbsolutePath()); - } - //Discard os metrics if the file has not been updated in the 5 second window. - if (opFileLastModified < startTime) { - return false; - } - //Only put data when opFile.lastModified() is newer than the lastUpdateTime in database. - //If there is an update, We'll delete existing data and insert new data. - if (lastUpdateTimePerTid.containsKey(threadID)) { - if (lastUpdateTimePerTid.get(threadID) == opFileLastModified) { - return false; - } - tidToDelete.add(threadID); - } - - String sOSMetrics = PerformanceAnalyzerMetrics.getMetric(opFile.getAbsolutePath()); - OSMetrics[] metrics = OSMetrics.values(); - for (OSMetrics metric : metrics) { - try { - String metricVal = PerformanceAnalyzerMetrics.extractMetricValue(sOSMetrics, metric.toString()); - if (metricVal != null) { - Double val = Double.parseDouble(metricVal); - osMetrics.put(metric.toString(), val); - } - } catch (Exception e) { - LOG.error(e, e); - LOG.error("Error parsing file - {} ExcepionCode: {},\n {}", - opFile.getAbsolutePath(), StatExceptionCode.READER_PARSER_ERROR.toString(), sOSMetrics); - StatsCollector.instance().logException(StatExceptionCode.READER_PARSER_ERROR); - throw e; - } - } - - String threadName = PerformanceAnalyzerMetrics.extractMetricValue(sOSMetrics, - OSMetricsCollector.MetaDataFields.threadName.toString()); - - int numMetrics = metrics.length + 3; - Object [] metricVals = new Object[numMetrics]; - metricVals[0] = threadID; - metricVals[1] = threadName; - for (int i = 2; i < numMetrics - 1; i++) { - metricVals[i] = osMetrics.get(metrics[i - 2].toString()); - } - metricVals[numMetrics - 1] = opFileLastModified; - - batchHandle.bind(metricVals); - return true; - } - - private void handleESMetrics(File esMetrics, String threadID, - long startTime, long endTime, BatchBindStep handle) { - String operation = esMetrics.getName(); //- shardBulk, shardSearch etc.. - for (File idFile : esMetrics.listFiles()) { - try { - handleidFile(idFile, threadID, startTime, endTime, operation, handle); - } catch (Exception e) { - LOG.error("Failed to parse ES Metrics with ExcepionCode: " + StatExceptionCode.READER_PARSER_ERROR.toString(), e); - StatsCollector.instance().logException(StatExceptionCode.READER_PARSER_ERROR); - } - } - } - - private String getPrimary(String primary) { - return primary == null ? "NA" : (primary.equals("true") ? "primary" : "replica"); - } - - private void emitStartMetric(String startMetrics, String rid, String threadId, - String operation, BatchBindStep handle) { - long st = Long.parseLong(PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, - ShardBulkMetric.START_TIME.toString())); - String indexName = PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, - ShardBulkDimension.INDEX_NAME.toString()); - String shardId = PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, - ShardBulkDimension.SHARD_ID.toString()); - String primary = getPrimary(PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, - ShardBulkDimension.PRIMARY.toString())); - String docCountString = PerformanceAnalyzerMetrics.extractMetricValue(startMetrics, - ShardBulkMetric.ITEM_COUNT.toString()); - long docCount = 0; - if (docCountString != null) { - docCount = Long.parseLong(docCountString); - } - handle.bind(shardId, indexName, rid, threadId, operation, primary, st, null, docCount); - } - - private void emitFinishMetric(String finishMetrics, String rid, String threadId, - String operation, BatchBindStep handle) { - long ft = Long.parseLong(PerformanceAnalyzerMetrics.extractMetricValue(finishMetrics, - ShardBulkMetric.FINISH_TIME.toString())); - String indexName = PerformanceAnalyzerMetrics.extractMetricValue(finishMetrics, - ShardBulkDimension.INDEX_NAME.toString()); - String shardId = PerformanceAnalyzerMetrics.extractMetricValue(finishMetrics, - ShardBulkDimension.SHARD_ID.toString()); - String primary = getPrimary(PerformanceAnalyzerMetrics.extractMetricValue(finishMetrics, - ShardBulkDimension.PRIMARY.toString())); - handle.bind(shardId, indexName, rid, threadId, operation, primary, null, ft, null); - } - - private void handleidFile(File idFile, String threadID, long startTime, - long endTime, String operation, BatchBindStep handle) { - String rid = idFile.getName(); - long lastModified = FileHelper.getLastModified(idFile, startTime, endTime); - if (lastModified < startTime || lastModified >= endTime) { - return; - } - for (File metricsFile: idFile.listFiles()) { - String metrics = PerformanceAnalyzerMetrics.getMetric(metricsFile.getAbsolutePath()); - try { - if (metricsFile.getName().equals(PerformanceAnalyzerMetrics.START_FILE_NAME)) { - emitStartMetric(metrics, rid, threadID, operation, handle); - } else if (metricsFile.getName().equals(PerformanceAnalyzerMetrics.FINISH_FILE_NAME)) { - emitFinishMetric(metrics, rid, threadID, operation, handle); - } - } catch (Exception e) { - LOG.error(e, e); - LOG.error("Error parsing file - {},\n {}", metricsFile.getAbsolutePath(), metrics); - LOG.error("Error parsing file - {} ExcepionCode: {},\n {}", - metricsFile.getAbsolutePath(), StatExceptionCode.READER_PARSER_ERROR.toString(), metrics); - StatsCollector.instance().logException(StatExceptionCode.READER_PARSER_ERROR); - } - } - } -} - - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/OSMetricsSnapshot.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/OSMetricsSnapshot.java deleted file mode 100644 index 27aa8e17..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/OSMetricsSnapshot.java +++ /dev/null @@ -1,482 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import java.sql.Connection; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jooq.BatchBindStep; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.SQLDialect; -import org.jooq.SelectField; -import org.jooq.SelectHavingStep; -import org.jooq.impl.DSL; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.OSMetrics; - -@SuppressWarnings("serial") -public class OSMetricsSnapshot implements Removable { - private static final Logger LOG = LogManager.getLogger(OSMetricsSnapshot.class); - - private final DSLContext create; - private final String tableName; - private Set dimensionColumns; - private static final String LAST_UPDATE_TIME_FIELD = "lastUpdateTime"; - - private static final LinkedHashSet METRIC_COLUMNS; - - public enum Fields { - tid, tName, weight - } - - static { - METRIC_COLUMNS = new LinkedHashSet<>(); - for (OSMetrics metric: OSMetrics.values()) { - METRIC_COLUMNS.add(metric.toString()); - } - } - - public DSLContext getDSLContext() { - return create; - } - - public OSMetricsSnapshot(Connection conn, String tableNamePrefix, Long windowEndTime) { - this.tableName = tableNamePrefix + windowEndTime; - this.create = DSL.using(conn, SQLDialect.SQLITE); - - this.dimensionColumns = new LinkedHashSet() { { - this.add(Fields.tid.toString()); - this.add(Fields.tName.toString()); - } }; - - LOG.debug("Creating a new os snapshot table - {}", tableName); - create - .createTable(this.tableName).columns(getFields()) - .execute(); - } - - public OSMetricsSnapshot(Connection conn, Long windowEndTime) { - this(conn, "os_", windowEndTime); - } - - public void putMetric(Map metrics, Map dimensions, long updateTime) { - Map, String> dimensionMap = new HashMap, String>(); - Map, Double> metricMap = new HashMap, Double>(); - Map, Long> updateTimeMap = new HashMap, Long>(); - - for (Map.Entry dimension: dimensions.entrySet()) { - dimensionMap.put(DSL.field( - DSL.name(dimension.getKey()), String.class), dimension.getValue()); - } - - for (Map.Entry metricName: metrics.entrySet()) { - metricMap.put(DSL.field( - DSL.name(metricName.getKey()), Double.class), metricName.getValue()); - } - - updateTimeMap.put(DSL.field(LAST_UPDATE_TIME_FIELD, Long.class), updateTime); - - create.insertInto(DSL.table(this.tableName)) - .set(metricMap) - .set(dimensionMap) - .set(updateTimeMap) - .execute(); - } - - public BatchBindStep startBatchPut() { - List dummyValues = new ArrayList<>(); - for (int i = 0; i < dimensionColumns.size(); i++) { - dummyValues.add(null); - } - for (int i = 0; i < METRIC_COLUMNS.size(); i++) { - dummyValues.add(null); - } - // last update time column - dummyValues.add(null); - return create.batch(create.insertInto(DSL.table(this.tableName)).values(dummyValues)); - } - - public void deleteByTid(List tids) { - create.delete(DSL.table(this.tableName)) - .where(DSL.field(Fields.tid.name(), String.class).in(tids)) - .execute(); - } - - public List> getMetricColumnFields() { - return OSMetricsSnapshot.METRIC_COLUMNS.stream().map(s -> DSL.field(s, Double.class)) - .collect(Collectors.toList()); - } - - public void putMetric(Map metrics, String tid, String tName) { - Map, Double> metricMap = new HashMap, Double>(); - - for (Map.Entry metricName: metrics.entrySet()) { - metricMap.put(DSL.field( - DSL.name(metricName.getKey()), Double.class), - metricName.getValue()); - } - - create.insertInto(DSL.table(this.tableName)) - .set(DSL.field(Fields.tid.toString()), tid) - .set(DSL.field(Fields.tName.toString()), tName) - .set(metricMap) - .execute(); - } - - public String getTableName() { - return this.tableName; - } - - public Result fetchAll() { - return create.select().from(DSL.table(this.tableName)).fetch(); - } - - public Result fetchNegative() { - return create.select().from(DSL.table(this.tableName)) - .where(DSL.field(OSMetrics.CPU_UTILIZATION.toString()).lt(0L)).fetch(); - } - - public SelectHavingStep selectAll() { - return create.select(getFields()).from(this.tableName); - } - - @Override - public void remove() { - LOG.info("Dropping {}", this.tableName); - create.dropTable(DSL.table(this.tableName)).execute(); - } - - public void logSnap() { - LOG.debug(() -> getDebugSnap()); - } - - public Result getDebugSnap() { - return create - .select(DSL.field(Fields.tid.toString()).as(Fields.tid.toString()), - DSL.field(Fields.tName.toString()).as(Fields.tName.toString()), - DSL.field(OSMetrics.CPU_UTILIZATION.toString()), - DSL.field(OSMetrics.PAGING_MIN_FLT_RATE.toString())) - .from(this.tableName) - .where(DSL.field(OSMetrics.CPU_UTILIZATION.toString(), - Double.class).ne(0d)) - .fetch(); - } - - public Result getOSMetrics() { - List> fields = new ArrayList>(); - fields.add(DSL.field(Fields.tid.toString()).as(Fields.tid.toString())); - fields.add(DSL.field(Fields.tName.toString()).as(Fields.tName.toString())); - for (String metricColumn: METRIC_COLUMNS) { - fields.add(DSL.field(metricColumn, Double.class).as(metricColumn)); - } - return create.select(fields) - .from(this.tableName) - .fetch(); - } - - public Map getLastUpdateTimePerTid() { - List> fields = new ArrayList>(); - fields.add(DSL.field(Fields.tid.name()).as(Fields.tid.name())); - fields.add(DSL.field(LAST_UPDATE_TIME_FIELD).as(LAST_UPDATE_TIME_FIELD)); - Result ret = create.select(fields) - .from(this.tableName) - .fetch(); - - Map lastUpdateTimePerTid = new HashMap<>(); - for (int i = 0; i < ret.size(); i ++) { - lastUpdateTimePerTid.put(ret.get(i).get(Fields.tid.name()).toString(), - Long.parseLong(ret.get(i).get(LAST_UPDATE_TIME_FIELD).toString())); - } - return lastUpdateTimePerTid; - } - - /** - * Given metrics in two windows calculates a new window which overlaps with the given windows. - * |------leftWindow-------|-------rightWindow--------| - * leftLastUpdateTime rightLastUpdateTime - * - * a b - * |-----alignedWindow-----| - * - * leftWeight = leftLastUpdateTime - a - * rightWeight = b - (rightLastUpdateTime - simpleInterval) - * - * This method assumes that both left/right windows are greater than or equal to 5 seconds. - * - * @param leftWindow a snapshot of the left window metrics - * @param rightWindow a snapshot of the right window metrics - * @param alignedWindow aligned window combinging left and right window - * @param a aligned window start time. - * @param b aligned window end time. - * - */ - public static void alignWindow(OSMetricsSnapshot leftWindow, - OSMetricsSnapshot rightWindow, String alignedWindow, - long a, long b) { - DSLContext create = leftWindow.getDSLContext(); - - String leftPrefix = "l_"; - String rightPrefix = "r_"; - - SelectHavingStep alignWindow = selectAlignWindow(create, - leftWindow.tableName, rightWindow.tableName, leftPrefix, rightPrefix); - - create.insertInto(DSL.table(alignedWindow)).select( - selectFieldsHasLeftAndRight(create, leftPrefix, rightPrefix, a, b, alignWindow) - .unionAll(selectFieldsHasLeftOnly(create, leftPrefix, rightPrefix, alignWindow)) - .unionAll(selectFieldsHasRightOnly(create, leftPrefix, rightPrefix, alignWindow)) - ).execute(); - } - - /** - * Select records that exists in both left window and right window. Calc result by its weight. - * - * MetricValue = ((l_updateTime - a) * l_Metric + (b - l_updateTime) * r_metric) / 5 - * - * Example: - * alignWindow: - * |tid|l_lastModifiTime|l_cpu|l_rss|r_lastModifiTime|r_cpu|r_rss| - * +---+----------------+-----+-----+----------------+-----+-----+ - * | 1| 3| 10| 10| | | | - * | 1| | | | 7| 20| 20| - * | 2| 4| 10| 10| | | | - * | 3| | | | 8| 10| 10| - * - * Return: - * |tid|lastModifiTime|cpu|rss| - * +---+--------------+---+---+ - * | 1| 3| 16| 16| - * - * @param leftPrefix field prefix when merge from left table to align table - * @param rightPrefix field prefix when merge from right table to align table - * @param alignWindow align window return from selectAlignWindow - * @return see above example - */ - private static SelectHavingStep selectFieldsHasLeftAndRight( - DSLContext create, - String leftPrefix, String rightPrefix, long a, long b, - SelectHavingStep alignWindow) { - ArrayList> fieldsHasLeftAndRight = new ArrayList>(); - fieldsHasLeftAndRight.add(DSL.field(Fields.tid.name()).as(Fields.tid.name())); - fieldsHasLeftAndRight.add(DSL.field(Fields.tName.name()).as(Fields.tName.name())); - for (String metricName: METRIC_COLUMNS) { - fieldsHasLeftAndRight.add( - DSL.field(leftPrefix + LAST_UPDATE_TIME_FIELD, Long.class).sub(a).mul(DSL.field(leftPrefix + metricName, Double.class)) - .add(DSL.val(b).sub(DSL.field(leftPrefix + LAST_UPDATE_TIME_FIELD, Long.class)) - .mul(DSL.field(rightPrefix + metricName, Double.class))) - .div(b - a).as(metricName)); - } - fieldsHasLeftAndRight.add(DSL.field(leftPrefix + LAST_UPDATE_TIME_FIELD).as(LAST_UPDATE_TIME_FIELD)); - - Condition conditionHasLeftAndRight = DSL.field(leftPrefix + LAST_UPDATE_TIME_FIELD, Long.class).isNotNull() - .and(DSL.field(rightPrefix + LAST_UPDATE_TIME_FIELD, Long.class).isNotNull()); - - return create.select(fieldsHasLeftAndRight).from(alignWindow).where(conditionHasLeftAndRight); - } - - /** - * Select records that only exists in the left window. - * - * Example: - * alignWindow: - * |tid|l_lastModifiTime|l_cpu|l_rss|r_lastModifiTime|r_cpu|r_rss| - * +---+----------------+-----+-----+----------------+-----+-----+ - * | 1| 3| 10| 10| | | | - * | 1| | | | 7| 20| 20| - * | 2| 4| 10| 10| | | | - * | 3| | | | 8| 10| 10| - * - * Return: - * |tid|lastModifiTime|cpu|rss| - * +---+--------------+---+---+ - * | 2| 4| 10| 10| - * - * @param leftPrefix field prefix when merge from left table to align table - * @param rightPrefix field prefix when merge from right table to align table - * @param alignWindow align window return from selectAlignWindow - * @return see above example - */ - private static SelectHavingStep selectFieldsHasLeftOnly( - DSLContext create, - String leftPrefix, String rightPrefix, - SelectHavingStep alignWindow) { - ArrayList> fieldsHasLeftOnly = new ArrayList>(); - fieldsHasLeftOnly.add(DSL.field(Fields.tid.name()).as(Fields.tid.name())); - fieldsHasLeftOnly.add(DSL.field(Fields.tName.name()).as(Fields.tName.name())); - for (String metricName: METRIC_COLUMNS) { - fieldsHasLeftOnly.add(DSL.field(leftPrefix + metricName, Double.class).as(metricName)); - } - fieldsHasLeftOnly.add(DSL.field(leftPrefix + LAST_UPDATE_TIME_FIELD).as(LAST_UPDATE_TIME_FIELD)); - - Condition conditionHasLeftOnly = DSL.field(leftPrefix + LAST_UPDATE_TIME_FIELD, Long.class).isNotNull() - .and(DSL.field(rightPrefix + LAST_UPDATE_TIME_FIELD, Long.class).isNull()); - - return create.select(fieldsHasLeftOnly).from(alignWindow).where(conditionHasLeftOnly); - } - - /** - * Select records that only exists in the right window. - * - * Example: - * alignWindow: - * |tid|l_lastModifiTime|l_cpu|l_rss|r_lastModifiTime|r_cpu|r_rss| - * +---+----------------+-----+-----+----------------+-----+-----+ - * | 1| 3| 10| 10| | | | - * | 1| | | | 7| 20| 20| - * | 2| 4| 10| 10| | | | - * | 3| | | | 8| 10| 10| - * - * Return: - * |tid|lastModifiTime|cpu|rss| - * +---+--------------+---+---+ - * | 3| | 10| 10| - * - * @param leftPrefix field prefix when merge from left table to align table - * @param rightPrefix field prefix when merge from right table to align table - * @param alignWindow align window return from selectAlignWindow - * @return see above example - */ - private static SelectHavingStep selectFieldsHasRightOnly( - DSLContext create, - String leftPrefix, String rightPrefix, - SelectHavingStep alignWindow) { - ArrayList> fieldsHasRightOnly = new ArrayList>(); - fieldsHasRightOnly.add(DSL.field(Fields.tid.name()).as(Fields.tid.name())); - fieldsHasRightOnly.add(DSL.field(Fields.tName.name()).as(Fields.tName.name())); - for (String metricName: METRIC_COLUMNS) { - fieldsHasRightOnly.add(DSL.field(rightPrefix + metricName, Double.class).as(metricName)); - } - fieldsHasRightOnly.add(DSL.field(leftPrefix + LAST_UPDATE_TIME_FIELD).as(LAST_UPDATE_TIME_FIELD)); - - Condition conditionHasRightOnly = DSL.field(leftPrefix + LAST_UPDATE_TIME_FIELD, Long.class).isNull() - .and(DSL.field(rightPrefix + LAST_UPDATE_TIME_FIELD, Long.class).isNotNull()); - - return create.select(fieldsHasRightOnly).from(alignWindow).where(conditionHasRightOnly); - } - - /** - * Merge left window table and right window table into align window. - * - * Example: - * Left window table - * |tid|lastModifiTime|cpu|rss| - * +---+--------------+---+---+ - * | 1| 3| 10| 10| - * | 2| 4| 10| 10| - * - * Right window table - * |tid|lastModifiTime|cpu|rss| - * +---+--------------+---+---+ - * | 1| 7| 20| 20| - * | 3| 8| 10| 10| - * - * Return align window - * |tid|l_lastModifiTime|l_cpu|l_rss|r_lastModifiTime|r_cpu|r_rss| - * +---+----------------+-----+-----+----------------+-----+-----+ - * | 1| 3| 10| 10| | | | - * | 1| | | | 7| 20| 20| - * | 2| 4| 10| 10| | | | - * | 3| | | | 8| 10| 10| - * - * @param create DSLContext - * @param leftTableName left table name - * @param rightTableName right table name - * @param leftPrefix field prefix when merge from left table to align table - * @param rightPrefix field prefix when merge from right table to align table - * @return see above example - */ - private static SelectHavingStep selectAlignWindow( - DSLContext create, String leftTableName, String rightTableName, String leftPrefix, String rightPrefix) { - List> fields = new ArrayList>(); - fields.add(DSL.field(Fields.tid.name(), String.class).as(Fields.tid.name())); - fields.add(DSL.field(Fields.tName.name(), String.class).as(Fields.tName.name())); - fields.add(DSL.max(DSL.field(leftPrefix + LAST_UPDATE_TIME_FIELD, Long.class)).as(leftPrefix + LAST_UPDATE_TIME_FIELD)); - for (String c: METRIC_COLUMNS) { - fields.add(DSL.max(DSL.field(leftPrefix + c, Double.class)).as(leftPrefix + c)); - } - fields.add(DSL.max(DSL.field(rightPrefix + LAST_UPDATE_TIME_FIELD, Long.class)).as(rightPrefix + LAST_UPDATE_TIME_FIELD)); - for (String c: METRIC_COLUMNS) { - fields.add(DSL.max(DSL.field(rightPrefix + c, Double.class)).as(rightPrefix + c)); - } - - return create.select(fields).from( - selectAlignWindowFromLeft(create, leftTableName, leftPrefix, rightPrefix) - .unionAll(selectAlignWindowFromRight(create, rightTableName, leftPrefix, rightPrefix)) - ).groupBy(DSL.field(Fields.tid.name(), String.class)); - } - - private static SelectHavingStep selectAlignWindowFromLeft( - DSLContext create, String tableName, String leftPrefix, String rightPrefix) { - List> fields = new ArrayList>(); - fields.add(DSL.field(Fields.tid.name(), String.class).as(Fields.tid.name())); - fields.add(DSL.field(Fields.tName.name(), String.class).as(Fields.tName.name())); - fields.add(DSL.field(LAST_UPDATE_TIME_FIELD, Long.class).as(leftPrefix + LAST_UPDATE_TIME_FIELD)); - for (String c : METRIC_COLUMNS) { - fields.add(DSL.field(c, Double.class).as(leftPrefix + c)); - } - fields.add(DSL.val(null, Long.class).as(rightPrefix + LAST_UPDATE_TIME_FIELD)); - for (String c : METRIC_COLUMNS) { - fields.add(DSL.val(null, Double.class).as(rightPrefix + c)); - } - return create.select(fields).from(tableName); - } - - private static SelectHavingStep selectAlignWindowFromRight( - DSLContext create, String tableName, String leftPrefix, String rightPrefix) { - List> fields = new ArrayList>(); - fields.add(DSL.field(Fields.tid.name(), String.class).as(Fields.tid.name())); - fields.add(DSL.field(Fields.tName.name(), String.class).as(Fields.tName.name())); - fields.add(DSL.val(null, Long.class).as(leftPrefix + LAST_UPDATE_TIME_FIELD)); - for (String c : METRIC_COLUMNS) { - fields.add(DSL.val(null, Double.class).as(leftPrefix + c)); - } - fields.add(DSL.field(LAST_UPDATE_TIME_FIELD, Long.class).as(rightPrefix + LAST_UPDATE_TIME_FIELD)); - for (String c : METRIC_COLUMNS) { - fields.add(DSL.field(c, Double.class).as(rightPrefix + c)); - } - return create.select(fields).from(tableName); - } - - public List> getFields() { - List> fields = new ArrayList>(); - for (String dimension: dimensionColumns) { - fields.add(DSL.field(dimension, String.class)); - } - for (String metric: METRIC_COLUMNS) { - fields.add(DSL.field(metric, Double.class)); - } - fields.add(DSL.field(DSL.name(LAST_UPDATE_TIME_FIELD), Long.class)); - return fields; - } - - public Set getMetricColumns() { - return OSMetricsSnapshot.METRIC_COLUMNS; - } -} - - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessor.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessor.java deleted file mode 100644 index 8d3127b2..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessor.java +++ /dev/null @@ -1,835 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; -import java.util.HashMap; -import java.util.Map; -import java.util.NavigableMap; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentSkipListMap; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.logging.log4j.util.Supplier; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.jooq.impl.DSL; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricName; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; -import com.google.common.annotations.VisibleForTesting; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; - -public class ReaderMetricsProcessor implements Runnable { - private static final Logger LOG = LogManager.getLogger(ReaderMetricsProcessor.class); - - private static final String DB_URL = "jdbc:sqlite:"; - private final Connection conn; - private final DSLContext create; - - //This semaphore is used to control access to metricsDBMap from threads outside of - //ReaderMetricsProcessor. - private NavigableMap metricsDBMap; - private NavigableMap osMetricsMap; - private NavigableMap shardRqMetricsMap; - private NavigableMap httpRqMetricsMap; - private NavigableMap masterEventMetricsMap; - private Map> nodeMetricsMap; - private static final int MAX_DATABASES = 2; - private static final int OS_SNAPSHOTS = 4; - private static final int RQ_SNAPSHOTS = 4; - private static final int HTTP_RQ_SNAPSHOTS = 4; - private static final int MASTER_EVENT_SNAPSHOTS = 4; - private final MetricsParser metricsParser; - private final String rootLocation; - private static final Map TIMING_STATS = new HashMap<>(); - private static final Map STATS_DATA = new HashMap<>(); - static { - STATS_DATA.put("MethodName", "ProcessMetrics"); - } - - private static ReaderMetricsProcessor current = null; - - public static void setCurrentInstance(ReaderMetricsProcessor currentInstance) { - current = currentInstance; - } - - public static ReaderMetricsProcessor getInstance() { - return current; - } - - public ReaderMetricsProcessor(String rootLocation) throws Exception { - conn = DriverManager.getConnection(DB_URL); - create = DSL.using(conn, SQLDialect.SQLITE); - metricsDBMap = new ConcurrentSkipListMap<>(); - osMetricsMap = new TreeMap<>(); - shardRqMetricsMap = new TreeMap<>(); - httpRqMetricsMap = new TreeMap<>(); - masterEventMetricsMap = new TreeMap<>(); - metricsParser = new MetricsParser(); - this.rootLocation = rootLocation; - - AllMetrics.MetricName[] names = AllMetrics.MetricName.values(); - nodeMetricsMap = new HashMap<>(names.length); - for (int i=0; i()); - } - } - - @Override - public void run() { - long startTime = System.currentTimeMillis(); - try { - Statement stmt = conn.createStatement(); - try { - stmt.executeUpdate("PRAGMA journal_mode = OFF"); - stmt.executeUpdate("PRAGMA soft_heap_limit = 10000000"); - } catch (Exception e) { - LOG.error("Unable to run PRAGMA"); - } finally { - stmt.close(); - } - - long runInterval = MetricsConfiguration.SAMPLING_INTERVAL / 2; - - while (true) { - //Create snapshots. - Statement vacuumStmt = conn.createStatement(); - try { - vacuumStmt.executeUpdate("VACUUM"); - } catch (Exception e) { - LOG.error("Unable to run Vacuum."); - } finally { - vacuumStmt.close(); - } - conn.setAutoCommit(false); - startTime = System.currentTimeMillis(); - //- Always read one sampling interval behind from current timestamp, - // otherwise reader may be trying to read bucket, which writer might not have written yet - ClusterLevelMetricsReader.collectNodeMetrics(startTime - MetricsConfiguration.SAMPLING_INTERVAL); - processMetrics(rootLocation, startTime); - trimOldSnapshots(); - conn.commit(); - conn.setAutoCommit(true); - long duration = System.currentTimeMillis() - startTime; - LOG.info("Total time taken: {}", duration); - if (duration < runInterval) { - Thread.sleep(runInterval - duration); - } - } - } catch (Throwable e) { - LOG.error( - (Supplier) () -> new ParameterizedMessage( - "READER PROCESSOR ERROR. NEEDS DEBUGGING {} ExceptionCode: {}.", - StatExceptionCode.OTHER.toString(), e.toString()), - e); - StatsCollector.instance().logException(); - - try { - long duration = System.currentTimeMillis() - startTime; - if (duration < MetricsConfiguration.SAMPLING_INTERVAL) { - Thread.sleep(MetricsConfiguration.SAMPLING_INTERVAL - duration); - } - } catch (Exception ex) { - LOG.error("Exception in sleep: {}", () -> ex); - } - throw new RuntimeException("READER ERROR"); - } finally { - try { - shutdown(); - LOG.error("Connection to the database was closed."); - } catch (Exception e) { - LOG.error("Unable to close all database connections and shutdown cleanly."); - } - } - } - - public void shutdown() { - try { - conn.close(); - } catch (Exception e) { - LOG.error("Unable to close inmemory database connection."); - } - - for (MetricsDB db: metricsDBMap.values()) { - try { - db.close(); - } catch (Exception e) { - LOG.error("Unable to close database - {}", db.getDBFilePath()); - } - } - } - - public void trimOldSnapshots() throws Exception { - trimMap(osMetricsMap, OS_SNAPSHOTS); - trimMap(shardRqMetricsMap, RQ_SNAPSHOTS); - trimMap(httpRqMetricsMap, HTTP_RQ_SNAPSHOTS); - trimMap(masterEventMetricsMap, MASTER_EVENT_SNAPSHOTS); - trimDatabases(metricsDBMap, MAX_DATABASES, PluginSettings.instance().shouldCleanupMetricsDBFiles()); - - for (NavigableMap snap : nodeMetricsMap - .values()) { - // do the same thing as OS_SNAPSHOTS. Eventually MemoryDBSnapshot - // will replace OSMetricsSnapshot as we want to our code to be - // stable. - trimMap(snap, OS_SNAPSHOTS); - } - } - - /** - * Deletes the lowest entries in the map till the size of the map is equal to maxSize. - */ - private void trimMap(NavigableMap map, int maxSize) throws Exception { - //Remove the oldest entries from the map - while (map.size() > maxSize) { - Map.Entry lowestEntry = map.firstEntry(); - if (lowestEntry != null) { - Removable value = (Removable) lowestEntry.getValue(); - value.remove(); - map.remove(lowestEntry.getKey()); - } - } - } - - /** - * Deletes the MetricsDB entries in the map till the size of the map is equal to maxSize. The actual on-disk - * files is deleted ony if the config is not set or set to true. - */ - public static void trimDatabases(NavigableMap map, int maxSize, boolean deleteDBFiles) throws Exception { - // Remove the oldest entries from the map, upto maxSize. - while (map.size() > maxSize) { - Map.Entry lowestEntry = map.firstEntry(); - if (lowestEntry != null) { - MetricsDB value = lowestEntry.getValue(); - map.remove(lowestEntry.getKey()); - value.remove(); - if (deleteDBFiles) { - value.deleteOnDiskFile(); - } - } - } - } - - /** - * Parse per thread OS metrics. - * OS metrics are generated per thread and written to files in - * /dev/shm/performanceanalyzer/{rotation_window}/threads/{tid}/os_metrics. - * This function parses the files written since the last successful run and populates an inmemory - * sqlite table with the results. A few metrics available are - cpu, rss, minor pagefaults etc. - * - * @param rootLocation where to find metric files - * @param startTime OSMetrics window start time - * @param endTime OSMetrics window end time - * @throws Exception thrown if the metric file could not be parsed correctly. - */ - public void parseOSMetrics(String rootLocation, long startTime, - long endTime) throws Exception { - long mCurrT = System.currentTimeMillis(); - if (osMetricsMap.get(endTime) == null) { - // handle last bucket before creating new bucket - if (osMetricsMap.lastEntry() != null) { - metricsParser.parseOSMetrics(rootLocation, startTime - MetricsConfiguration.SAMPLING_INTERVAL, - startTime, osMetricsMap.lastEntry().getValue()); - } - - LOG.info("Creating OsSnap {}", endTime); - OSMetricsSnapshot osSnap = new OSMetricsSnapshot(this.conn, "os_", endTime); - osMetricsMap.put(endTime, osSnap); - } - OSMetricsSnapshot osSnap = osMetricsMap.get(endTime); - - metricsParser.parseOSMetrics(rootLocation, startTime, endTime, osSnap); - - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for parsing OS Metrics: {}", mFinalT - mCurrT); - TIMING_STATS.put("parseOSMetrics", (double)(mFinalT - mCurrT)); - } - - /** - * Parse per node metric. Node level metrics are generated per node This - * function parses metrics files written since the last successful run and - * populates an in-memory sqlite table with the results. A few metrics - * available are - Starting memory limit for overall parent-level breaker, - * maximum limit defined for the survivor memory pool (-1 means there is - * no maxium limit) etc. - * - * @param currTimestamp when reader starts processing metric files - * @throws Exception thrown if we have issues parsing metrics - */ - public void parseNodeMetrics(long currTimestamp) - throws Exception { - long mCurrT = System.currentTimeMillis(); - LOG.info("Creating MemoryDBSnapshot {}", currTimestamp); - for (Map.Entry> entry : - nodeMetricsMap.entrySet()) { - AllMetrics.MetricName name = entry.getKey(); - NavigableMap currMap = entry.getValue(); - - MemoryDBSnapshot currSnap = new MemoryDBSnapshot(this.conn, name, - currTimestamp); - - // parse metrics for last window. - Map.Entry lastNodeMetricsEntry = currMap - .lastEntry(); - long lastNodeMetricsSnapshotTime = 0; - if (lastNodeMetricsEntry == null) { - lastNodeMetricsSnapshotTime = 0L; - } else { - lastNodeMetricsSnapshotTime = lastNodeMetricsEntry.getKey(); - } - - MetricProperties currParser = MetricPropertiesConfig.getInstance() - .getProperty(name); - - if (currParser.dispatch( - currSnap, currTimestamp, lastNodeMetricsSnapshotTime)) { - LOG.info( - "Adding new {} snapshot- currTimestamp {}, actualTime {}", - name, currTimestamp, currSnap.getLastUpdatedTime()); - currMap.put(currSnap.getLastUpdatedTime(), currSnap); - } else { - LOG.info("Did not add values into {} snapshot. Clearing it {}", - name, currTimestamp); - currSnap.remove(); - } - - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for parsing {} Metrics: {}", name, - mFinalT - mCurrT); - } - - } - - /** - * Parse all events generated by shard requests. - * This function processes shard events such as shardBulk and shardSearch. Every operation - * emits a start event and an end event. The events can be found in - * /dev/shm/performanceanalyzer/{rotation_window}/threads/{tid}/{operation}/{rid}/. The start event data is written to - * a file called start and the end event data is in a file called end. - * - * @param rootLocation where to find metric files - * @param currWindowStartTime the start time of current sampling period. - * The bound of the period where that value is measured is - * MetricsConfiguration.SAMPLING_INTERVAL. - * @param currWindowEndTime the end time of current sampling period. - * The bound of the period where that value is measured is - * MetricsConfiguration.SAMPLING_INTERVAL. - * - * @throws Exception thrown if we have issues parsing metrics - */ - public void parseRequestMetrics(String rootLocation, long currWindowStartTime, - long currWindowEndTime) throws Exception { - long mCurrT = System.currentTimeMillis(); - if (shardRqMetricsMap.get(currWindowStartTime) == null) { - ShardRequestMetricsSnapshot rqSnap = new ShardRequestMetricsSnapshot(this.conn, currWindowStartTime); - Map.Entry entry = shardRqMetricsMap.lastEntry(); - if (entry != null) { - rqSnap.rolloverInflightRequests(entry.getValue()); - } - metricsParser.parseRequestMetrics(rootLocation, currWindowStartTime, - currWindowEndTime, rqSnap); - - LOG.debug(() -> rqSnap.fetchAll()); - shardRqMetricsMap.put(currWindowStartTime, rqSnap); - LOG.info("Adding new RQ snapshot- currWindowStartTime {}", currWindowStartTime); - } - - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for parsing Request Metrics: {}", mFinalT - mCurrT); - TIMING_STATS.put("parseRequestMetrics", (double)(mFinalT - mCurrT)); - } - - - /** - * Parse all http events generated by shard requests. - * This function processes http events such as bulk and search. Every operation - * emits a start event and an end event. The events can be found in - * /dev/shm/performanceanalyzer/{rotation_window}/threads/{tid}/http/{operation}/{rid}/. The start event data - * is written to * a file called start and the end event data is in a file called end. - * - * @param rootLocation where to find metric files - * @param currWindowStartTime the start time of current sampling period. - * The bound of the period where that value is measured is - * MetricsConfiguration.SAMPLING_INTERVAL. - * @param currWindowEndTime the end time of current sampling period. - * The bound of the period where that value is measured is - * MetricsConfiguration.SAMPLING_INTERVAL. - * - * @throws Exception thrown if we have issues parsing metrics - */ - public void parseHttpRequestMetrics(String rootLocation, long currWindowStartTime, - long currWindowEndTime) throws Exception { - long mCurrT = System.currentTimeMillis(); - if (httpRqMetricsMap.get(currWindowStartTime) == null) { - HttpRequestMetricsSnapshot httpRqSnap = new HttpRequestMetricsSnapshot(this.conn, currWindowStartTime); - Map.Entry entry = httpRqMetricsMap.lastEntry(); - if (entry != null) { - httpRqSnap.rolloverInflightRequests(entry.getValue()); - } - metricsParser.parseHttpMetrics(rootLocation, currWindowStartTime, - currWindowEndTime, httpRqSnap); - LOG.debug(() -> httpRqSnap.fetchAll()); - httpRqMetricsMap.put(currWindowStartTime, httpRqSnap); - LOG.info("Adding new HTTP RQ snapshot- currTimestamp {}", currWindowStartTime); - } - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for parsing HTTP Request Metrics: {}", mFinalT - mCurrT); - TIMING_STATS.put("parseHttpRequestMetrics", (double)(mFinalT - mCurrT)); - } - - /** - * Enrich event data with OS metrics and calculate aggregated metrics on dimensions like (shard, index, operation, role). - * We emit metrics for the previous window interval as we need two metric windows to align OSMetrics. - * Ex: To emit metrics between 5-10, we need OSMetrics emitted at 8 and 13, to be able to calculate the - * metrics correctly. The aggregated metrics are then written to a metricsDB. - * - * @param currWindowStartTime the start time of current sampling period. - * The bound of the period where that value is measured is - * MetricsConfiguration.SAMPLING_INTERVAL. - * @throws Exception thrown if we have issues parsing metrics - */ - public void emitMetrics(long currWindowStartTime) throws Exception { - long prevWindowStartTime = currWindowStartTime - MetricsConfiguration.SAMPLING_INTERVAL; - - if (metricsDBMap.get(prevWindowStartTime) != null) { - LOG.info("The metrics for this timestamp already exist. Skipping."); - return; - } - - long mCurrT = System.currentTimeMillis(); - //This is object holds a reference to the temporary os snapshot. It is used to delete tables at the end of this - //reader cycle. The OSMetricsSnapshot expects windowEndTime in the constructor. - OSMetricsSnapshot alignedOSSnapHolder = new OSMetricsSnapshot(this.conn, "os_aligned_", - currWindowStartTime); - OSMetricsSnapshot osAlignedSnap = alignOSMetrics(prevWindowStartTime, - prevWindowStartTime + MetricsConfiguration.SAMPLING_INTERVAL, alignedOSSnapHolder); - - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for aligning OS Metrics: {}", mFinalT - mCurrT); - - mCurrT = System.currentTimeMillis(); - MetricsDB metricsDB = createMetricsDB(prevWindowStartTime); - - emitMasterMetrics(prevWindowStartTime, metricsDB); - emitShardRequestMetrics(prevWindowStartTime, alignedOSSnapHolder, osAlignedSnap, metricsDB); - emitHttpRequestMetrics(prevWindowStartTime, metricsDB); - emitNodeMetrics(currWindowStartTime, metricsDB); - - metricsDB.commit(); - metricsDBMap.put(prevWindowStartTime, metricsDB); - mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for emitting Metrics: {}", mFinalT - mCurrT); - TIMING_STATS.put("emitMetrics", (double)(mFinalT - mCurrT)); - } - - private void emitHttpRequestMetrics(long prevWindowStartTime, MetricsDB metricsDB) throws Exception { - - if (httpRqMetricsMap.containsKey(prevWindowStartTime)) { - - HttpRequestMetricsSnapshot prevHttpRqSnap = httpRqMetricsMap.get(prevWindowStartTime); - MetricsEmitter.emitHttpMetrics(create, metricsDB, prevHttpRqSnap); - }else { - LOG.info("Http request snapshot for the previous window does not exist. Not emitting metrics."); - } - } - - private void emitShardRequestMetrics(long prevWindowStartTime, - OSMetricsSnapshot alignedOSSnapHolder, - OSMetricsSnapshot osAlignedSnap, - MetricsDB metricsDB) throws Exception { - - if (shardRqMetricsMap.containsKey(prevWindowStartTime)) { - - ShardRequestMetricsSnapshot preShardRequestMetricsSnapshot = shardRqMetricsMap.get(prevWindowStartTime); - MetricsEmitter.emitWorkloadMetrics(create, metricsDB, preShardRequestMetricsSnapshot); // calculate latency - if (osAlignedSnap != null) { - MetricsEmitter.emitAggregatedOSMetrics(create, metricsDB, osAlignedSnap, preShardRequestMetricsSnapshot); // table join - MetricsEmitter.emitThreadNameMetrics(create, metricsDB, osAlignedSnap); // threads other than bulk and query - } - alignedOSSnapHolder.remove(); - }else { - LOG.info("Shard request snapshot for the previous window does not exist. Not emitting metrics."); - } - } - - private void emitMasterMetrics(long prevWindowStartTime, MetricsDB metricsDB) { - - if (masterEventMetricsMap.containsKey(prevWindowStartTime)) { - - MasterEventMetricsSnapshot preMasterEventSnapshot = masterEventMetricsMap.get(prevWindowStartTime); - MetricsEmitter.emitMasterEventMetrics(metricsDB, preMasterEventSnapshot); - }else { - LOG.info("Master snapshot for the previous window does not exist. Not emitting metrics."); - } - } - - private void parseMasterEventMetrics(String rootLocation, long currWindowStartTime, long currWindowEndTime) { - - long mCurrT = System.currentTimeMillis(); - if (masterEventMetricsMap.get(currWindowStartTime) == null) { - MasterEventMetricsSnapshot masterEventMetricsSnapshot = new MasterEventMetricsSnapshot(conn, currWindowStartTime); - Map.Entry entry = masterEventMetricsMap.lastEntry(); - - if (entry != null) { - masterEventMetricsSnapshot.rolloverInflightRequests(entry.getValue()); - } - - metricsParser.parseMasterEventMetrics(rootLocation, currWindowStartTime, - currWindowEndTime, masterEventMetricsSnapshot); - LOG.debug(() -> masterEventMetricsSnapshot.fetchAll()); - masterEventMetricsMap.put(currWindowStartTime, masterEventMetricsSnapshot); - LOG.info("Adding new Master Event snapshot- currTimestamp {}", currWindowStartTime); - } - - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for parsing Master Event Metrics: {}", mFinalT - mCurrT); - TIMING_STATS.put("parseMasterEventMetrics", (double)(mFinalT - mCurrT)); - } - - public void processMetrics(String rootLocation, long currTimestamp) throws Exception { - TIMING_STATS.clear(); - long start = System.currentTimeMillis(); - parseNodeMetrics(currTimestamp); - long currWindowEndTime = PerformanceAnalyzerMetrics.getTimeInterval(currTimestamp, MetricsConfiguration.SAMPLING_INTERVAL); - long currWindowStartTime = currWindowEndTime - MetricsConfiguration.SAMPLING_INTERVAL; - parseOSMetrics(rootLocation, currWindowEndTime, currWindowEndTime + MetricsConfiguration.SAMPLING_INTERVAL); - parseRequestMetrics(rootLocation, currWindowStartTime, currWindowEndTime); - parseHttpRequestMetrics(rootLocation, currWindowStartTime, currWindowEndTime); - parseMasterEventMetrics(rootLocation, currWindowStartTime, currWindowEndTime); - emitMetrics(currWindowStartTime); - StatsCollector.instance().logStatsRecord(null, STATS_DATA, TIMING_STATS, start, System.currentTimeMillis()); - } - - /** - * Returns per thread OSMetrics between startTime and endTime. - * OSMetrics might have been collected for windows that dont completely overlap with startTime and endTime. - * This function calculates the weighted average of metrics in each overlapping window and sums them up to find - * the average metrics in the requested window. - * - * @param startTime the start time of the previous sampling period. - * The bound of the period where that value is measured is - * MetricsConfiguration.SAMPLING_INTERVAL. - * @param endTime the end time of the previous sampling period. - * The bound of the period where that value is measured is - * MetricsConfiguration.SAMPLING_INTERVAL. - * @param alignedWindow where we store aligned snapshot - * - * @return - * alignedWindow if we have two sampled snapshot; - * a sampled snapshot if we have only one sampled snapshot within - * startTime and endTime; - * null if the number of total snapshots is less than OS_SNAPSHOTS or - * if there is no snapshot taken after startTime or - * right window snapshot ends at or before endTime - * - * @throws Exception thrown when we have issues in aligning window - */ - public OSMetricsSnapshot alignOSMetrics(long startTime, long endTime, OSMetricsSnapshot alignedWindow) - throws Exception { - LOG.info("Aligning metrics for {}, {}", startTime, endTime); - //Find osmetric windows that overlap with the expected window. - //This is atmost 2 but maybe less than 2. If less than 2, simply return the existing window. - - //If we have insufficient snapshots just return - if (osMetricsMap.size() < OS_SNAPSHOTS) { - LOG.warn("Exited due to too few snapshots - {}", osMetricsMap.size()); - return null; - } - - Map.Entry entry = osMetricsMap.higherEntry(startTime); - //There is no snapshot taken after startTime. - if (entry == null) { - LOG.warn("No OS snapshot above startTime."); - return null; - } - - //Start time of the previous snapshot. - Long t1 = entry.getKey(); - if (t1 == null) { - LOG.error("We dont have an OS snapshot above startTime."); - return null; - } - //Next higher key. - Long t2 = osMetricsMap.higherKey(t1); - - if (t2 == null) { - LOG.error("We dont have the next OS snapshot above startTime."); - return entry.getValue(); - } - - if (t2 < endTime) { - LOG.error("Right window snapshot ends before endTime. rw: {}, lw: {}, startTime: {}, endTime: {}", - t2, t1, startTime, endTime); - //TODO: As a quick fix we ignore this window. We might want to consider multiple windows instead. - return null; - } - - LOG.info("Adding new scaled OS snapshot- actualTime {}", startTime); - OSMetricsSnapshot leftWindow = osMetricsMap.get(t1); - OSMetricsSnapshot rightWindow = osMetricsMap.get(t2); - OSMetricsSnapshot.alignWindow(leftWindow, rightWindow, alignedWindow.getTableName(), - startTime, endTime); - return alignedWindow; - } - - /** - * Returns per node metrics between startTime and endTime. - * These metrics might have been collected for windows that dont completely - * overlap with startTime and endTime. - * This function calculates the weighted average of metrics in each - * overlapping window and sums them up to find the average metrics in the - * requested window. - * - * So in the code, startTime is "a" below, endTime is "b" below. Reader - * window is [a, b]. We want to find "x", the cut-off point between two - * writer window. - * - * Given metrics in two writer windows calculates a new reader window which - * overlaps with the given windows. - * |------leftWindow-------|-------rightWindow--------| - * x - * a b - * |-----------alignedWindow------| - * - * - * We are emitting aligned metrics for previous window, not current window. - * This is to make sure we have two windows to align. Otherwise, if we - * emit metrics for current window, we might not have two writer window - * metrics. - * - * If this is the time line: - * - * + writer writes to the left window at 2000l - * + reader reads at 6000l - * + writer writes to the right window at 7000l - * + reader reads at 11000l - * Then according to PerformanceAnalyzerMetrics.getTimeInterval, the previous reader - * window is [0, 5000], current reader window is [5000, 10000]. - * - * If we align for current reader window, we need writer window ends in - * 7000l and 12000l. But we don't have 12000l at 11000l. - * - * @param metricName the name of the metric we want to align - * @param metricMap the in-memory database for this metric - * @param readerStartTime the start time of the previous sampling period. - * The bound of the period where that value is measured is - * MetricsConfiguration.SAMPLING_INTERVAL. - * @param readerEndTime the end time of the previous sampling period. - * The bound of the period where that value is measured is - * MetricsConfiguration.SAMPLING_INTERVAL. - * @param alignedWindow where we store aligned snapshot - * - * @return - * alignedWindow if we have two sampled snapshot; - * a sampled snapshot if we have only one sampled snapshot within - * startTime and endTime; - * null if the number of total snapshots is less than OS_SNAPSHOTS or - * if there is no snapshot taken after startTime or - * right window snapshot ends at or before endTime - * - * @throws Exception thrown when we have issues in aligning window - */ - public MemoryDBSnapshot alignNodeMetrics(AllMetrics.MetricName metricName, - NavigableMap metricMap, long readerStartTime, - long readerEndTime, MemoryDBSnapshot alignedWindow) throws Exception { - - LOG.info("Aligning node metrics for {}, from {} to {}", metricName, readerStartTime, - readerEndTime); - // Find metric windows that overlap with the expected window. - // This is at most 2 but maybe less than 2. If less than 2, simply - // return the existing window. - - // If we have insufficient snapshots just return - // We need left writer window, right writer window. Also since we are - // dealing with previous reader window, we need at least 3 snapshots. - if (metricMap.size() < 3) { - LOG.warn("Exited node metrics for {}, due to too few snapshots", metricName); - return null; - } - - // retrieve a snapshot ending at t1 = x - Map.Entry entry = metricMap - .ceilingEntry(readerStartTime); - // There is no snapshot taken after startTime. - if (entry == null) { - LOG.warn("No {} metrics snapshot above startTime.", metricName); - return null; - } - - // Start time of the previous snapshot. - Long t1 = entry.getKey(); - if (t1 == null) { - LOG.error("We dont have an {} snapshot above startTime.", metricName); - return null; - } - // Next higher key representing the end time of the rightWindow above - - Long t2 = metricMap.higherKey(t1); - - if (t2 == null) { - LOG.error("We dont have the next {} snapshot above startTime.", - metricName); - return entry.getValue(); - } - - // t1 and startTime are already aligned. Just return the snapshot - // between t2 and t1. - if (t1 == readerStartTime) { - LOG.info("Found matching {} snapshot.", metricName); - return metricMap.get(t2); - } - - if (t2 <= readerEndTime) { - LOG.error( - "Right window {} snapshot ends at or before endTime. rw: {}, lw: {}, startTime: {}, endTime: {}", - metricName, t2, t1, readerStartTime, readerEndTime); - //TODO: As a quick fix we ignore this window. We might want to consider multiple windows instead. - return null; - } - - LOG.info("Adding new scaled {} snapshot- actualTime {}", metricName, - readerStartTime); - // retrieve left and right window using osMetricsMap, whose key is the - // largest last modification time. We use values in the future to - // represent values in the past. So if at t1, writer writes values 1, - // the interval [t1-sample interval, t1] has value 1. - MemoryDBSnapshot leftWindow = metricMap.get(t1); - MemoryDBSnapshot rightWindow = metricMap.get(t2); - - alignedWindow.alignWindow(leftWindow, rightWindow, t1, readerStartTime, - readerEndTime); - return alignedWindow; - } - - public Connection getConnection() { - return this.conn; - } - - public DSLContext getDSLContext() { - return this.create; - } - - /** - * This is called by operations outside of the ReaderMetricsProcessor. - * - * @return the latest on-disk database - */ - public Map.Entry getMetricsDB() { - //If metricsDBMap is being trimmed we wait and acquire the latest - return metricsDBMap.lastEntry(); - } - - public MetricsDB createMetricsDB(long timestamp) throws Exception { - MetricsDB db = new MetricsDB(timestamp); - return db; - } - - public void deleteDBs() throws Exception { - for (MetricsDB db: metricsDBMap.values()) { - db.remove(); - } - } - - /** - * Enrich event data with node metrics and calculate aggregated metrics on - * dimensions like (shard, index, operation, role). The aggregated metrics - * are then written to a metricsDB. - * - * @param currWindowStartTime the start time of current sampling period. - * The bound of the period where that value is measured is - * MetricsConfiguration.SAMPLING_INTERVAL. - * @param metricsDB on-disk database to which we want to emit metrics - * - * @throws Exception if we have issues emitting or aligning metrics - */ - public void emitNodeMetrics(long currWindowStartTime, MetricsDB metricsDB) - throws Exception { - long prevWindowStartTime = currWindowStartTime - - MetricsConfiguration.SAMPLING_INTERVAL; - - for (Map.Entry> entry : nodeMetricsMap - .entrySet()) { - - MetricName metricName = entry.getKey(); - - NavigableMap metricMap = entry.getValue(); - - if (metricMap.get(prevWindowStartTime) != null) { - LOG.info( - "The metrics in {} for this timestamp already exist. Skipping.", - metricName); - return; - } - - long mCurrT = System.currentTimeMillis(); - - // This is object holds a reference to the temporary memory db - // snapshot. It is used to delete tables at the end of this - // reader cycle. - - MemoryDBSnapshot alignedSnapshotHolder = new MemoryDBSnapshot( - getConnection(), metricName, currWindowStartTime, true); - MemoryDBSnapshot alignedSnapshot = alignNodeMetrics(metricName, - metricMap, prevWindowStartTime, currWindowStartTime, - alignedSnapshotHolder); - - long mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for aligning {} Metrics: {}", metricName, - mFinalT - mCurrT); - - if (alignedSnapshot == null) { - alignedSnapshotHolder.remove(); - LOG.info( - "{} snapshot for the previous window does not exist. Not emitting metrics.", - metricName); - continue; - } - - mCurrT = System.currentTimeMillis(); - MetricsEmitter.emitNodeMetrics(create, metricsDB, alignedSnapshot); - - // alignedSnapshotHolder cannot be the left or right window we are - // trying to align, so we can safely remove. - alignedSnapshotHolder.remove(); - - mFinalT = System.currentTimeMillis(); - LOG.info("Total time taken for emitting node metrics: {}", - mFinalT - mCurrT); - } - } - - @VisibleForTesting - Map> getNodeMetricsMap() { - return nodeMetricsMap; - } - - @VisibleForTesting - void putNodeMetricsMap(AllMetrics.MetricName name, - NavigableMap metricsMap) { - this.nodeMetricsMap.put(name, metricsMap); - } -} - - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/Removable.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/Removable.java deleted file mode 100644 index 5f65ba2f..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/Removable.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -/** - * Interface that should be implemented by snapshot holders that need to be trimmed. - */ -public interface Removable { - void remove() throws Exception; -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ShardRequestMetricsSnapshot.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ShardRequestMetricsSnapshot.java deleted file mode 100644 index ef6ec419..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ShardRequestMetricsSnapshot.java +++ /dev/null @@ -1,479 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import java.sql.Connection; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jooq.BatchBindStep; -import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Record2; -import org.jooq.Result; -import org.jooq.SQLDialect; -import org.jooq.SelectField; -import org.jooq.SelectHavingStep; -import org.jooq.impl.DSL; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.DBUtils; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CommonDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardBulkMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardOperationMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; - -/** - * Snapshot of start/end events generated by per shard operations like shardBulk/shardSearch. - */ -@SuppressWarnings("serial") -public class ShardRequestMetricsSnapshot implements Removable { - private static final Logger LOG = LogManager.getLogger(ShardRequestMetricsSnapshot.class); - - private static final ArrayList> groupByRidOp = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.RID.name()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.name()), String.class)); - } }; - - private final DSLContext create; - private final Long windowStartTime; - private final String tableName; - private static final Long EXPIRE_AFTER = 600000L; - private List> columns; - - public enum Fields { - SHARD_ID(CommonDimension.SHARD_ID.toString()), - INDEX_NAME(CommonDimension.INDEX_NAME.toString()), - RID(HttpRequestMetricsSnapshot.Fields.RID.toString()), - TID("tid"), - OPERATION(CommonDimension.OPERATION.toString()), - SHARD_ROLE(CommonDimension.SHARD_ROLE.toString()), - ST(HttpRequestMetricsSnapshot.Fields.ST.toString()), - ET(HttpRequestMetricsSnapshot.Fields.ET.toString()), - LAT(HttpRequestMetricsSnapshot.Fields.LAT.toString()), - TUTIL("tUtil"), - TTIME("ttime"), - LATEST("latest"), - DOC_COUNT(ShardBulkMetric.DOC_COUNT.toString()); - - private final String fieldValue; - - Fields(String fieldValue) { - this.fieldValue = fieldValue; - } - - @Override - public String toString() { - return fieldValue; - } - }; - - public ShardRequestMetricsSnapshot(Connection conn, Long windowStartTime) throws Exception { - this.create = DSL.using(conn, SQLDialect.SQLITE); - this.windowStartTime = windowStartTime; - this.tableName = "shard_rq_" + windowStartTime; - - //The order of names specified for bulk inserts needs to match the order of the columns specified here. - this.columns = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.SHARD_ID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDEX_NAME.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.RID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.TID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.SHARD_ROLE.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.ST.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.ET.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.DOC_COUNT.toString()), Long.class)); - } }; - - create.createTable(this.tableName) - .columns(this.columns) - .execute(); - } - - public void putStartMetric(Long startTime, Map dimensions) { - Map, String> dimensionMap = new HashMap, String>(); - for (Map.Entry dimension: dimensions.entrySet()) { - dimensionMap.put(DSL.field( - DSL.name(dimension.getKey()), String.class), dimension.getValue()); - } - create.insertInto(DSL.table(this.tableName)) - .set(DSL.field(DSL.name(Fields.ST.toString()), Long.class), startTime) - .set(dimensionMap) - .execute(); - } - - public BatchBindStep startBatchPut() { - List dummyValues = new ArrayList<>(); - for (int i = 0; i < columns.size(); i++) { - dummyValues.add(null); - } - return create.batch(create.insertInto(DSL.table(this.tableName)).values(dummyValues)); - } - - public void putEndMetric(Long endTime, Map dimensions) { - Map, String> dimensionMap = new HashMap, String>(); - for (Map.Entry dimension: dimensions.entrySet()) { - dimensionMap.put(DSL.field( - DSL.name(dimension.getKey()), String.class), dimension.getValue()); - } - create.insertInto(DSL.table(this.tableName)) - .set(DSL.field(DSL.name(Fields.ET.toString()), Long.class), endTime) - .set(dimensionMap) - .execute(); - } - - public Result fetchAll() { - return create.select().from(DSL.table(this.tableName)).fetch(); - } - - /** - * Return per request latency. - * - * Actual Table - * |shard|indexName|rid |tid |operation |role| st| et| - * +-----+---------+-------+----+----------+----+-------------+-------------+ - * |0 |sonested |2447782|7069|shardquery|NA | {null}|1535065340625| - * |0 |sonested |2447782|7069|shardquery|NA |1535065340330| {null}| - * |0 |sonested |2447803|7069|shardfetch|NA | {null}|1535065344730| - * |0 |sonested |2447803|7069|shardfetch|NA |1535065344729| {null}| - * |0 |sonested |2447781|7069|shardfetch|NA |1535065340227| {null}| - * - * Latency Table - * |shard|indexName|rid |tid |operation |role| st| et| lat| - * +-----+---------+-------+----+----------+----+-------------+-------------+-----+ - * |0 |sonested |2447782|7069|shardquery|NA |1535065340330|1535065340625| 255| - * |0 |sonested |2447803|7069|shardfetch|NA |1535065344729|1535065344730| 001| - * - * @return rows with latency of each shard request - */ - public SelectHavingStep fetchLatency() { - - List> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.SHARD_ID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDEX_NAME.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.RID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.TID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.SHARD_ROLE.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.ST.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.ET.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.DOC_COUNT.toString()), Long.class)); - this.add(DSL.field(Fields.ET.toString()).minus(DSL.field(Fields.ST.toString())).as(DSL.name(Fields.LAT.toString()))); - } }; - - - return create.select(fields).from(groupByRidOpSelect()) - .where(DSL.field(Fields.ET.toString()).isNotNull().and( - DSL.field(Fields.ST.toString()).isNotNull())); - } - - /** - * Return per operation latency. - * This is a performance optimization to avoid writing one entry per request back into metricsDB. - * This function returns one row per operation. - * - * Latency Table - * |shard|indexName|rid |tid |operation |role| st| et| lat| - * +-----+---------+-------+----+----------+----+-------------+-------------+-----+ - * |0 |sonested |2447782|7069|shardquery|NA |1535065340330|1535065340625| 255| - * |0 |sonested |2447783|7069|shardquery|NA |1535065340330|1535065340635| 265| - * |0 |sonested |2447803|7069|shardfetch|NA |1535065344729|1535065344730| 001| - * |0 |sonested |2447804|7069|shardfetch|NA |1535065344729|1535065344732| 003| - * - * Returned Table - * |shard|indexName|tid |operation |role|sum_lat|avg_lat|min_lat|max_lat| - * +-----+---------+----+----------+----+-------------+-------------+-------+-------+-------+-------+ - * |0 |sonested |7069|shardquery|NA | 520| 260| 255| 265| - * |0 |sonested |7069|shardfetch|NA | 004| 002| 001| 003| - * - * @return aggrated latency by ShardID, IndexName, Operation, and ShardRole. - */ - public Result fetchLatencyByOp() { - ArrayList> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.SHARD_ID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDEX_NAME.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.SHARD_ROLE.toString()), String.class)); - this.add(DSL.sum(DSL.field(DSL.name(Fields.LAT.toString()), Double.class)) - .as(DBUtils.getAggFieldName(Fields.LAT.toString(), MetricsDB.SUM))); - this.add(DSL.avg(DSL.field(DSL.name(Fields.LAT.toString()), Double.class)) - .as(DBUtils.getAggFieldName(Fields.LAT.toString(), MetricsDB.AVG))); - this.add(DSL.min(DSL.field(DSL.name(Fields.LAT.toString()), Double.class)) - .as(DBUtils.getAggFieldName(Fields.LAT.toString(), MetricsDB.MIN))); - this.add(DSL.max(DSL.field(DSL.name(Fields.LAT.toString()), Double.class)) - .as(DBUtils.getAggFieldName(Fields.LAT.toString(), MetricsDB.MAX))); - this.add(DSL.count().as(ShardOperationMetric.SHARD_OP_COUNT.toString())); - this.add(DSL.sum(DSL.field(DSL.name(Fields.DOC_COUNT.toString()), Double.class)) - .as(ShardBulkMetric.DOC_COUNT.toString())); - } }; - - ArrayList> groupByFields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.SHARD_ID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDEX_NAME.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.SHARD_ROLE.toString()), String.class)); - } }; - - return create.select(fields).from(fetchLatency()) - .groupBy(groupByFields).fetch(); - } - - /** - * Return row per request. - * - * Actual Table - * |shard|indexName|rid |tid |operation |role| st| et| - * +-----+---------+-------+----+----------+----+-------------+-------------+ - * |0 |sonested |2447782|7069|shardquery|NA | {null}|1535065340625| - * |0 |sonested |2447782|7069|shardquery|NA |1535065340330| {null}| - * |0 |sonested |2447803|7069|shardfetch|NA | {null}|1535065344730| - * |0 |sonested |2447803|7069|shardfetch|NA |1535065344729| {null}| - * |0 |sonested |2447781|7069|shardfetch|NA |1535065340227| {null}| - * - * Latency Table - * windowStartTime = 1535065340330 - * endTime = 1535065345330 - * |shard|indexName|rid |tid |operation |role| st| et| - * +-----+---------+-------+----+----------+----+-------------+-------------+ - * |0 |sonested |2447782|7069|shardquery|NA |1535065340330|1535065340625| - * |0 |sonested |2447803|7069|shardfetch|NA |1535065344729|1535065344730| - * |0 |sonested |2447781|7069|shardfetch|NA |1535065340227|1535065345330| - * - * @return aggregated latency rows for each shard request - */ - - public SelectHavingStep getCoalescedRequestsForTimeSpentInWindow() { - Long endTime = this.windowStartTime + MetricsConfiguration.SAMPLING_INTERVAL; - ArrayList> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.SHARD_ID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDEX_NAME.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.RID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.TID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.SHARD_ROLE.toString()), String.class)); - } }; - - fields.add(DSL.greatest(DSL.coalesce(DSL.max(DSL.field(Fields.ST.toString())), - (this.windowStartTime)), this.windowStartTime) - .as(DSL.name(Fields.ST.toString()))); - fields.add(DSL.least(DSL.coalesce(DSL.max(DSL.field(Fields.ET.toString())), endTime), endTime) - .as(DSL.name(Fields.ET.toString()))); - - return create.select(fields).from(DSL.table(this.tableName)) - .groupBy(groupByRidOp); - } - - public SelectHavingStep getTimeSpentPerRequest() { - ArrayList> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.SHARD_ID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDEX_NAME.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.RID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.TID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.SHARD_ROLE.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.ST.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.ET.toString()), Long.class)); - } }; - - fields.add(DSL.field(Fields.ET.toString()).minus(DSL.field(Fields.ST.toString())).as(DSL.name(Fields.LAT.toString()))); - return create.select(fields).from(getCoalescedRequestsForTimeSpentInWindow()); - } - - public SelectHavingStep groupByRidOpSelect() { - ArrayList> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.SHARD_ID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDEX_NAME.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.RID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.TID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.SHARD_ROLE.toString()), String.class)); - this.add(DSL.max(DSL.field(Fields.DOC_COUNT.toString())).as(DSL.name(Fields.DOC_COUNT.toString()))); - this.add(DSL.max(DSL.field(Fields.ST.toString())).as(DSL.name(Fields.ST.toString()))); - this.add(DSL.max(DSL.field(Fields.ET.toString())).as(DSL.name(Fields.ET.toString()))); - } }; - - - return create.select(fields) - .from(DSL.table(this.tableName)) - .groupBy(groupByRidOp); - } - - public SelectHavingStep requestsPerThreadSelect() { - SelectHavingStep groupByRidOp = groupByRidOpSelect(); - List> fields = new ArrayList>() { { - - this.add(DSL.field(DSL.name(Fields.SHARD_ID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDEX_NAME.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.RID.toString()), String.class)); - this.add(DSL.field(groupByRidOp.field(Fields.TID.toString()))); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.SHARD_ROLE.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.ST.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.ET.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.DOC_COUNT.toString()), Double.class)); - this.add(DSL.field(DSL.name(Fields.LATEST.toString()), Long.class)); - } }; - SelectHavingStep> threadTable = create - .select(DSL.max(DSL.field(Fields.ST.toString(), Long.class)).as(Fields.LATEST.toString()), - DSL.field(DSL.name(Fields.TID.toString()), String.class).as(Fields.TID.toString())) - .from(groupByRidOp) - .groupBy(DSL.field(Fields.TID.toString())); - - return create.select(fields).from(groupByRidOp).join(threadTable) - .on(threadTable.field(DSL.field(Fields.TID.toString())).eq(groupByRidOp.field(Fields.TID.toString()))); - } - - /** - * Fetch inflight requests, and ignore missing events. - * The intention of this function is to identify requests that have a missing event and are no longer inflight. - * Once, we identify such requests we simply ignore them in all metrics calculation. The key invariant of this function - * is the fact that at any time there is a single active request on a thread. Hence, if we see more than one active - * request on a thread we ignore all requests on that thread except the latest one. - * - * Actual Table - * |shard|indexName|rid |tid |operation |role| st| et| - * +-----+---------+-------+----+----------+----+-------------+-------------+ - * |0 |sonested |2447781|7069|shardfetch|NA |1535065340227| {null}| - * |0 |sonested |2447782|7069|shardquery|NA | {null}|1535065340625| - * |0 |sonested |2447782|7069|shardquery|NA |1535065340330| {null}| - * |0 |sonested |2447803|7069|shardfetch|NA | {null}|1535065344730| - * |0 |sonested |2447803|7069|shardfetch|NA |1535065344729| {null}| - * - * Intermediate select - * |shard|indexName|rid |tid |operation |role| st| et| latest| - * +-----+---------+-------+----+----------+----+-------------+-------------+-------------+ - * |0 |sonested |2447781|7069|shardfetch|NA |1535065340227| {null}|1535065344729| - * |0 |sonested |2447782|7069|shardquery|NA |1535065340330|1535065340625|1535065344729| - * |0 |sonested |2447803|7069|shardfetch|NA |1535065344729|1535065344730|1535065344729| - * - * windowStartTime = 1535065340330 - * We ignore the first row as it is lower than the current window and we have new requests executing on the same thread. - * - * |shard|indexName|rid |tid |operation |role| st| et| - * +-----+---------+-------+----+----------+----+-------------+-------------+ - * |0 |sonested |2447782|7069|shardquery|NA |1535065340330|1535065340625| - * |0 |sonested |2447803|7069|shardfetch|NA |1535065344729|1535065344730| - * - * @return fetched inflight requests - */ - public SelectHavingStep fetchInflightSelect() { - ArrayList> fields = new ArrayList>() { { - this.add(DSL.field(DSL.name(Fields.SHARD_ID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.INDEX_NAME.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.RID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.TID.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.OPERATION.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.SHARD_ROLE.toString()), String.class)); - this.add(DSL.field(DSL.name(Fields.ST.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.ET.toString()), Long.class)); - this.add(DSL.field(DSL.name(Fields.DOC_COUNT.toString()), Long.class)); - } }; - - SelectHavingStep reqPerThread = requestsPerThreadSelect(); - - return create.select(fields).from(reqPerThread) - .where(DSL.field(Fields.ST.toString()).isNotNull() - .and(DSL.field(Fields.ST.toString()).gt(this.windowStartTime) - .or(DSL.field(Fields.LATEST.toString()).eq(DSL.field(Fields.ST.toString()))) - ) - .and(DSL.field(Fields.ET.toString()).isNull()) - .and(DSL.field(Fields.ST.toString()).gt(this.windowStartTime - EXPIRE_AFTER))); - } - - - public SelectHavingStep fetchTotalTimeTable(SelectHavingStep timeSpentPerRequestSelect) { - List> fields = new ArrayList>(); - fields.add(DSL.field(Fields.TID.toString())); - fields.add(DSL.sum(DSL.field(Fields.LAT.toString(), Double.class)).as(Fields.TTIME.toString())); - return create.select(fields) - .from(timeSpentPerRequestSelect) - .groupBy(DSL.field(Fields.TID.toString())); - } - - public Result fetchThreadUtilizationRatio() { - return create.select() - .from(fetchThreadUtilizationRatioTable()) - .fetch(); - } - - /** - * Calculate the percentage of time spent on a thread by each request in the current time window. - * - * Latency Table - * |shard|indexName|rid |tid |operation |role| st| et| lat| - * +-----+---------+-------+----+----------+----+-------------+-------------+-----+ - * |0 |sonested |2447782|7069|shardquery|NA |1535065340330|1535065340625| 255| - * |0 |sonested |2447783|7069|shardquery|NA |1535065340330|1535065340635| 265| - * |0 |sonested |2447803|7069|shardfetch|NA |1535065344729|1535065344730| 001| - * |0 |sonested |2447804|7069|shardfetch|NA |1535065344729|1535065344732| 003| - - * ThreadUtilizationTable - * ttime = (255+265+001+003) - * tUtil = lat/ttime - * |shard|indexName|rid |tid |operation |role| st| et| lat|ttime| tUtil| - * +-----+---------+-------+----+----------+----+-------------+-------------+-----+-----+-----+ - * |0 |sonested |2447782|7069|shardquery|NA |1535065340330|1535065340625| 255| 524|0.4866| - * |0 |sonested |2447783|7069|shardquery|NA |1535065340330|1535065340635| 265| 524|0.5057| - * |0 |sonested |2447803|7069|shardfetch|NA |1535065344729|1535065344730| 001| 524|0.0019| - * |0 |sonested |2447804|7069|shardfetch|NA |1535065344729|1535065344732| 003| 524|0.0058 - * - * @return thread utilization table - */ - public SelectHavingStep fetchThreadUtilizationRatioTable() { - ArrayList> requestAndTotalThreadTimeFields = new ArrayList>(); - SelectHavingStep timeSpentPerReq = getTimeSpentPerRequest(); - SelectHavingStep threadTable = fetchTotalTimeTable(timeSpentPerReq); - requestAndTotalThreadTimeFields.addAll(Arrays.asList(timeSpentPerReq.fields())); - requestAndTotalThreadTimeFields.add(threadTable.field(Fields.TTIME.toString())); - SelectHavingStep requestAndTotalThreadTimeSelect = create - .select(requestAndTotalThreadTimeFields) - .from(timeSpentPerReq) - .join(threadTable) - .on(timeSpentPerReq.field(Fields.TID.toString(), String.class).eq( - threadTable.field(Fields.TID.toString(), String.class))); - - ArrayList> tUtilFields = new ArrayList>(); - tUtilFields.addAll(Arrays.asList(requestAndTotalThreadTimeSelect.fields())); - tUtilFields.add(requestAndTotalThreadTimeSelect.field(Fields.LAT.toString()).mul(DSL.val(1.0d)) - .div(requestAndTotalThreadTimeSelect.field(Fields.TTIME.toString(), Double.class)).as(Fields.TUTIL.toString())); - return create.select(tUtilFields).from(requestAndTotalThreadTimeSelect); - } - - public String getTableName() { - return this.tableName; - } - - @Override - public void remove() { - create.dropTable(DSL.table(this.tableName)).execute(); - } - - public void rolloverInflightRequests(ShardRequestMetricsSnapshot prevSnap) { - create.insertInto(DSL.table(this.tableName)).select( - prevSnap.fetchInflightSelect() - ).execute(); - LOG.debug("Inflight shard requests"); - LOG.debug(() -> fetchAll()); - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rest/MetricsHandler.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rest/MetricsHandler.java deleted file mode 100644 index 906685cf..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rest/MetricsHandler.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.rest; - -import java.util.Map; -import java.util.HashMap; - -class MetricsHandler { - protected Map getParamsMap(String query) { - Map result = new HashMap<>(); - if (query != null) { - for (String param : query.split("&")) { - String[] entry = param.split("="); - if (entry.length > 1) { - result.put(entry[0], entry[1]); - } else { - result.put(entry[0], ""); - } - } - } - return result; - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rest/QueryMetricsRequestHandler.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rest/QueryMetricsRequestHandler.java deleted file mode 100644 index 87972940..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rest/QueryMetricsRequestHandler.java +++ /dev/null @@ -1,353 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.rest; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.security.InvalidParameterException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.logging.log4j.util.Supplier; -import org.jooq.Record; -import org.jooq.Result; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.model.MetricAttributes; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.model.MetricsModel; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader.ClusterLevelMetricsReader; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader.ReaderMetricsProcessor; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.JsonConverter; -import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; -import javax.net.ssl.HttpsURLConnection; - -/** - * Request handler that supports querying MetricsDB on every EC2 instance. - * Example query – "http://localhost:9600/_metricsdb?metrics=cpu,rss,memory%20agg=sum,avg,sum%20dims=index,operation,shard." - * We can fetch multiple metrics using this interface and also specify the dimensions/aggregations for fetching the metrics. - * We create a new metricsDB every 5 seconds and API only supports querying the latest snapshot. - */ -public class QueryMetricsRequestHandler extends MetricsHandler implements HttpHandler { - - private static final Logger LOG = LogManager.getLogger(QueryMetricsRequestHandler.class); - private static final int HTTP_CLIENT_CONNECTION_TIMEOUT = 200; - - public QueryMetricsRequestHandler() { - } - - @Override - public void handle(HttpExchange exchange) throws IOException { - String requestMethod = exchange.getRequestMethod(); - LOG.info("{} {} {}", exchange.getRequestMethod(), exchange.getRemoteAddress(), exchange.getRequestURI()); - ReaderMetricsProcessor mp = ReaderMetricsProcessor.getInstance(); - if (mp == null) { - sendResponse(exchange, - "{\"error\":\"Metrics Processor is not initialized. The reader has run into an issue or has just started.\"}", - HttpURLConnection.HTTP_UNAVAILABLE); - - LOG.warn("Metrics Processor is not initialized. The reader has run into an issue or has just started."); - return; - } - - Map.Entry dbEntry = mp.getMetricsDB(); - if (dbEntry == null) { - sendResponse(exchange, - "{\"error\":\"There are no metrics databases. The reader has run into an issue or has just started.\"}", - HttpURLConnection.HTTP_UNAVAILABLE); - - LOG.warn("There are no metrics databases. The reader has run into an issue or has just started."); - return; - } - MetricsDB db = dbEntry.getValue(); - Long dbTimestamp = dbEntry.getKey(); - - if (requestMethod.equalsIgnoreCase("GET")) { - LOG.debug("Query handler called."); - - if (isUnitLookUp(exchange)) { - getMetricUnits(exchange); - return; - } - - Map params = getParamsMap(exchange.getRequestURI().getQuery()); - - exchange.getResponseHeaders().set("Content-Type", "application/json"); - - try { - List metricList = parseArrayParam(params, "metrics", false); - List aggList = parseArrayParam(params, "agg", false); - List dimList = parseArrayParam(params, "dim", true); - - if (metricList.size() != aggList.size()) { - sendResponse(exchange, - "{\"error\":\"metrics/aggregations should have the same number of entries.\"}", - HttpURLConnection.HTTP_BAD_REQUEST); - return; - } - - if (!validParams(exchange, metricList, dimList, aggList)) { - return; - } - - String nodes = params.get("nodes"); - String response = collectStats(db, dbTimestamp, metricList, aggList, dimList, nodes); - sendResponse(exchange, response, HttpURLConnection.HTTP_OK); - } catch (InvalidParameterException e) { - LOG.error("DB file path : {}", db.getDBFilePath()); - LOG.error( - (Supplier) () -> new ParameterizedMessage( - "QueryException {} ExceptionCode: {}.", - e.toString(), StatExceptionCode.REQUEST_ERROR.toString()), - e); - StatsCollector.instance().logException(StatExceptionCode.REQUEST_ERROR); - String response = "{\"error\":\"" + e.getMessage() + "\"}"; - sendResponse(exchange, response, HttpURLConnection.HTTP_BAD_REQUEST); - } catch (Exception e) { - LOG.error("DB file path : {}", db.getDBFilePath()); - LOG.error( - (Supplier) () -> new ParameterizedMessage( - "QueryException {} ExceptionCode: {}.", - e.toString(), StatExceptionCode.REQUEST_ERROR.toString()), - e); - StatsCollector.instance().logException(StatExceptionCode.REQUEST_ERROR); - String response = "{\"error\":\"" + e.toString() + "\"}"; - sendResponse(exchange, response, HttpURLConnection.HTTP_INTERNAL_ERROR); - } - } else { - exchange.sendResponseHeaders(HttpURLConnection.HTTP_NOT_FOUND, -1); - exchange.close(); - } - } - - private boolean isUnitLookUp(HttpExchange exchange) throws IOException { - if (exchange.getRequestURI().toString().equals(PerformanceAnalyzerApp.QUERY_URL + "/units")) { - return true; - } - return false; - } - - private void getMetricUnits(HttpExchange exchange) throws IOException { - Map metricUnits = new HashMap<>(); - for (Map.Entry entry : MetricsModel.ALL_METRICS.entrySet()) { - String metric = entry.getKey(); - String unit = entry.getValue().unit; - metricUnits.put(metric, unit); - } - sendResponse(exchange, JsonConverter.writeValueAsString(metricUnits), HttpURLConnection.HTTP_OK); - } - - public boolean validParams(HttpExchange exchange, List metricList, List dimList, List aggList) - throws IOException { - for (String metric : metricList) { - if (MetricsModel.ALL_METRICS.get(metric) == null) { - sendResponse(exchange, - String.format("{\"error\":\"%s is an invalid metric.\"}", metric), - HttpURLConnection.HTTP_BAD_REQUEST); - return false; - } else { - for (String dim : dimList) { - if (!MetricsModel.ALL_METRICS.get(metric).dimensionNames.contains(dim)) { - sendResponse(exchange, - String.format("{\"error\":\"%s is an invalid dimension for %s metric.\"}", - dim, metric), - HttpURLConnection.HTTP_BAD_REQUEST); - return false; - } - } - } - } - for (String agg : aggList) { - if (!MetricsDB.AGG_VALUES.contains(agg)) { - sendResponse(exchange, String.format("{\"error\":\"%s is an invalid aggregation type.\"}", agg), - HttpURLConnection.HTTP_BAD_REQUEST); - return false; - } - } - - return true; - } - - public List parseArrayParam(Map params, String name, boolean optional) { - if (!optional) { - if (!params.containsKey(name) || params.get(name).isEmpty()) { - throw new InvalidParameterException(String.format("%s parameter needs to be set", name)); - } - } - - if (params.containsKey(name) && !params.get(name).isEmpty()) { - return Arrays.asList(params.get(name).split(",")); - } - return new ArrayList<>(); - } - - public void sendResponse(HttpExchange exchange, String response, int status) throws IOException { - try (OutputStream os = exchange.getResponseBody()) { - exchange.sendResponseHeaders(status, response.length()); - os.write(response.getBytes()); - } catch (Exception e) { - response = e.toString(); - exchange.sendResponseHeaders(HttpURLConnection.HTTP_INTERNAL_ERROR, response.length()); - } - } - - public String getParamString(List metricList, List aggList, - List dimList) { - String metricString = "metrics=" + String.join(",", metricList); - String aggString = "agg=" + String.join(",", aggList); - String dimString = "dim=" + String.join(",", dimList); - return String.join("&", metricString, aggString, dimString); - } - - public String collectStats(MetricsDB db, Long dbTimestamp, List metricList, - List aggList, List dimList, String nodeParam) throws Exception { - String localResponse = ""; - if (db != null) { - Result metricResult = db.queryMetric( - metricList, aggList, dimList); - if (metricResult == null) { - localResponse = "{}"; - } else { - localResponse = metricResult.formatJSON(); - } - } else { - //Empty JSON. - localResponse = "{}"; - } - String localResponseWithTimestamp = getQueryJsonWithTimestamp(dbTimestamp, localResponse); - - if (nodeParam == null) { - return localResponseWithTimestamp; - } - - if (nodeParam.equals("all")) { - LOG.debug("Collecting metrics from all nodes"); - HashMap nodeResponses = new HashMap<>(); - String params = getParamString(metricList, aggList, dimList); - ClusterLevelMetricsReader.NodeDetails[] nodes = ClusterLevelMetricsReader.getNodes(); - String localNodeId = "local"; - if (nodes.length != 0) { - localNodeId = nodes[0].getId(); - } - nodeResponses.put(localNodeId, localResponseWithTimestamp); - for (int i = 1; i < nodes.length; i++) { - ClusterLevelMetricsReader.NodeDetails node = nodes[i]; - LOG.debug("Collecting remote stats"); - try { - String remoteNodeStats = collectRemoteStats(node.getHostAddress(), - PerformanceAnalyzerApp.QUERY_URL, - params - ); - nodeResponses.put(node.getId(), remoteNodeStats); - } catch (Exception e) { - LOG.error("Unable to collect stats for node, addr:{}, exception: {} ExceptionCode: {}", - node.getHostAddress(), e, StatExceptionCode.REQUEST_REMOTE_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.REQUEST_REMOTE_ERROR); - } - } - String response = nodeJsonBuilder(nodeResponses); - LOG.debug("Returned the final text - \n{}", response); - return response; - } - return localResponseWithTimestamp; - } - - public String getQueryJsonWithTimestamp(Long timestamp, String queryResponse) { - return String.format("{\"timestamp\": %d, \"data\": %s}", timestamp, queryResponse); - } - - public String nodeJsonBuilder(HashMap nodeResponses) { - StringBuilder outputJson = new StringBuilder(); - outputJson.append("{"); - Set nodeSet = nodeResponses.keySet(); - String[] nodes = nodeSet.toArray(new String[nodeSet.size()]); - if (nodes.length > 0) { - outputJson.append("\""); - outputJson.append(nodes[0]); - outputJson.append("\": "); - outputJson.append(nodeResponses.get(nodes[0])); - } - - for (int i = 1; i < nodes.length; i++) { - outputJson.append(", \""); - outputJson.append(nodes[i]); - outputJson.append("\" :"); - outputJson.append(nodeResponses.get(nodes[i])); - } - - outputJson.append("}"); - return outputJson.toString(); - } - - protected String collectRemoteStats(String nodeIP, String uri, String queryString) throws Exception { - HttpURLConnection conn = getUrlConnection(nodeIP, uri, queryString); - - conn.setConnectTimeout(HTTP_CLIENT_CONNECTION_TIMEOUT); - int responseCode = conn.getResponseCode(); - if (responseCode != HttpURLConnection.HTTP_OK) { - LOG.error("Did not receive 200 from remote node. NodeIP-{} ExceptionCode: {}", - nodeIP, StatExceptionCode.REQUEST_REMOTE_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.REQUEST_REMOTE_ERROR); - throw new Exception("Did not receive a 200 response code from the remote node."); - } - - BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); - StringBuilder response = new StringBuilder(); - String inputLine; - try { - while ((inputLine = in.readLine()) != null) { - response.append(inputLine); - } - } finally { - in.close(); - } - - return response.toString(); - } - - private HttpURLConnection getUrlConnection(String nodeIP, String uri, String queryString) throws IOException { - boolean httpsEnabled = PluginSettings.instance().getHttpsEnabled(); - String protocol = "http"; - if (httpsEnabled) { - protocol = "https"; - } - String urlString = String.format("%s://%s:9600%s?%s", protocol, nodeIP, uri, queryString); - LOG.debug("Remote URL - {}", urlString); - URL url = new URL(urlString); - - if (httpsEnabled) { - return (HttpsURLConnection) url.openConnection(); - } else { - return (HttpURLConnection) url.openConnection(); - } - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/FileHelper.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/FileHelper.java deleted file mode 100644 index d77d747b..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/FileHelper.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.util; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; - -public class FileHelper { - private static final Logger log = LogManager.getLogger(FileHelper.class); - private static boolean jvmSupportMillisecondFileModityTime = true; - private static long SECOND_TO_MILLISECONDS = 1000; - - static { - try { - // Create tmp file and test if we can read millisecond - for (int i = 0; i < 2; i ++) { - File tmpFile = File.createTempFile("performanceanalyzer", ".tmp"); - tmpFile.deleteOnExit(); - jvmSupportMillisecondFileModityTime = tmpFile.lastModified() % 1000 != 0; - if (jvmSupportMillisecondFileModityTime) { - break; - } - Thread.sleep(2); - } - } catch (Exception ex) { - log.error("Having issue creating tmp file. Using default value.", ex); - } - log.info("jvmSupportMillisecondFileModityTime: {}", jvmSupportMillisecondFileModityTime); - } - - public static long getLastModified(File file, long startTime, long endTime) { - if (!file.isFile() || jvmSupportMillisecondFileModityTime) { - return file.lastModified(); - } - - if (file.lastModified() < startTime - SECOND_TO_MILLISECONDS || - file.lastModified() > endTime) { - return file.lastModified(); - } - - try (BufferedReader reader = new BufferedReader(new FileReader(file))) { - String line = reader.readLine(); - if (line != null) { - String[] fields = line.split(PerformanceAnalyzerMetrics.sKeyValueDelimitor); - if (fields[0].equals(PerformanceAnalyzerMetrics.METRIC_CURRENT_TIME)) { - return Long.parseLong(fields[1]); - } - } - } catch (Exception ex) { - StatsCollector.instance().logException(); - log.debug("Having issue to read current time from the content of file. Using file metadata; excpetion: {} ExceptionCode: {}", - () -> ex, () -> StatExceptionCode.OTHER.toString()); - } - return file.lastModified(); - } -} - diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/JsonConverter.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/JsonConverter.java deleted file mode 100644 index f3cb56dd..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/JsonConverter.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.util; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.fasterxml.jackson.core.JsonGenerationException; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatExceptionCode; - - -public class JsonConverter { - private static final ObjectMapper MAPPER = new ObjectMapper(); - - private static final Logger LOG = LogManager.getLogger(JsonConverter.class); - - - /** - * We can miss writing a metric if exception is thrown. - * @param value a Java object - * @return the converted string from the input Java object - */ - public static String writeValueAsString(Object value) { - try { - return MAPPER.writeValueAsString(value); - } catch (JsonGenerationException e) { - LOG.warn("Json generation error " + e.getMessage()); - throw new IllegalArgumentException(e); - } catch (JsonMappingException e) { - LOG.warn("Json Mapping Error: " + e.getMessage()); - throw new IllegalArgumentException(e); - } catch (IOException e) { - LOG.warn("IO error: " + e.getMessage()); - throw new IllegalArgumentException(e); - } - } - - public static Map createMapFrom(String json) { - - try { - if (json.trim().length() != 0) { - return MAPPER.readValue(json, - new TypeReference>() { }); - } - } catch (IOException e) { - LOG.debug("IO error: {} for json {} with ExceptionCode: {}", - () -> e.toString(), () -> json, () -> StatExceptionCode.JSON_PARSER_ERROR.toString()); - StatsCollector.instance().logException(StatExceptionCode.JSON_PARSER_ERROR); - } - return Collections.emptyMap(); - } - - /** - * Search a Jackson JsonNode inside a JSON string matching the input path - * expression - * - * @param jsonString - * an encoded JSON string - * @param paths - * path fragments - * @return the matching Jackson JsonNode or null in case of no match. - * @throws IOException if underlying input contains invalid content of type - * JsonParser supports - * @throws JsonProcessingException if underlying input contains invalid - * content of type JsonParser supports - * @throws IOException if underlying input contains invalid content of type - * JsonParser supports - */ - public static JsonNode getChildNode(String jsonString, String... paths) - throws JsonProcessingException, IOException { - JsonNode rootNode = MAPPER.readTree(jsonString); - return getChildNode(rootNode, paths); - } - - /** - * Search a Jackson JsonNode inside a Jackson JsonNode matching the input - * path expression - * - * @param jsonNode - * a Jackson JsonNode - * @param paths - * path fragments - * @return the matching Jackson JsonNode or null in case of no match. - */ - public static JsonNode getChildNode(JsonNode jsonNode, String... paths) { - for (int i = 0; i < paths.length; i++) { - String path = paths[i]; - if (!jsonNode.has(path)) { - return null; - } - - jsonNode = jsonNode.get(path); - } - - return jsonNode; - } - - /** - * Search a long number inside a JSON string matching the input path - * expression - * - * @param jsonString - * an encoded JSON string - * @param paths - * path fragments - * @return the matching long number or null in case of no match. - * - * @throws JsonPathNotFoundException thrown if the input path is invalid - * @throws IOException thrown if underlying input contains invalid content - * of type JsonParser supports - * @throws JsonProcessingException thrown if underlying input contains - * invalid content of type JsonParser supports - */ - public static long getLongValue(String jsonString, String... paths) - throws JsonPathNotFoundException, JsonProcessingException, - IOException { - JsonNode jsonNode = getChildNode(jsonString, paths); - if (jsonNode != null) { - return jsonNode.longValue(); - } - throw new JsonPathNotFoundException(); - } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/JsonPathNotFoundException.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/JsonPathNotFoundException.java deleted file mode 100644 index 764c8b8c..00000000 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/JsonPathNotFoundException.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.util; - -public class JsonPathNotFoundException extends Exception { - public JsonPathNotFoundException() { } -} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/Utils.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/Utils.java new file mode 100644 index 00000000..e36e00eb --- /dev/null +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/Utils.java @@ -0,0 +1,24 @@ +package com.amazon.opendistro.elasticsearch.performanceanalyzer.util; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.CircuitBreakerCollector; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceEventMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NodeDetailsCollector; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.NodeStatsMetricsCollector; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ThreadPoolMetricsCollector; + +public class Utils { + + public static void configureMetrics() { + MetricsConfiguration.MetricConfig cdefault = MetricsConfiguration.cdefault ; + MetricsConfiguration.CONFIG_MAP.put(CircuitBreakerCollector.class, cdefault); + MetricsConfiguration.CONFIG_MAP.put(ThreadPoolMetricsCollector.class, cdefault); + MetricsConfiguration.CONFIG_MAP.put(NodeDetailsCollector.class, cdefault); + MetricsConfiguration.CONFIG_MAP.put(NodeStatsMetricsCollector.class, cdefault); + MetricsConfiguration.CONFIG_MAP.put(MasterServiceEventMetrics.class, new MetricsConfiguration.MetricConfig(1000, 0, 0)); + MetricsConfiguration.CONFIG_MAP.put(MasterServiceMetrics.class, cdefault); + + } + +} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/writer/EventLogQueueProcessor.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/writer/EventLogQueueProcessor.java new file mode 100644 index 00000000..45c86d12 --- /dev/null +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/writer/EventLogQueueProcessor.java @@ -0,0 +1,148 @@ +package com.amazon.opendistro.elasticsearch.performanceanalyzer.writer; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.http_action.config.PerformanceAnalyzerConfigAction; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.Event; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.EventLogFileHandler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public class EventLogQueueProcessor { + private static final Logger LOG = LogManager.getLogger( + EventLogQueueProcessor.class); + + private final ScheduledExecutorService writerExecutor = + Executors.newScheduledThreadPool(1); + private final EventLogFileHandler eventLogFileHandler; + private final long initialDelayMillis; + private final long purgePeriodicityMillis; + private long lastTimeBucket; + + public EventLogQueueProcessor(EventLogFileHandler eventLogFileHandler, + long initialDelayMillis, + long purgePeriodicityMillis) { + this.eventLogFileHandler = eventLogFileHandler; + this.initialDelayMillis = initialDelayMillis; + this.purgePeriodicityMillis = purgePeriodicityMillis; + this.lastTimeBucket = 0; + } + + public void scheduleExecutor() { + ScheduledFuture futureHandle = + writerExecutor.scheduleAtFixedRate(this::purgeQueueAndPersist, + // The initial delay is critical here. The collector threads + // start immediately with the Plugin. This thread purges the + // queue and writes data to file. So, it waits for one run of + // the collectors to complete before it starts, so that the + // queue has elements to drain. + initialDelayMillis, + purgePeriodicityMillis, TimeUnit.MILLISECONDS); + new Thread(() -> { + try { + futureHandle.get(); + } catch (InterruptedException e) { + LOG.error("Scheduled execution was interrupted", e); + } catch (CancellationException e) { + LOG.warn("Watcher thread has been cancelled", e); + } catch (ExecutionException e) { + LOG.error("QueuePurger interrupted. Caused by ", + e.getCause()); + } + }).start(); + } + + // This executes every purgePeriodicityMillis interval. + public void purgeQueueAndPersist() { + // Return if the writer is not enabled. + if (PerformanceAnalyzerConfigAction.getInstance() == null) { + return; + } else if (!PerformanceAnalyzerConfigAction.getInstance().isFeatureEnabled()) { + // If PA is disabled, then we return as we don't want to generate + // new files. But we also want to drain the queue so that when it is + // enabled next, we don't have the current elements as they would be + // old. + if (PerformanceAnalyzerMetrics.metricQueue.size() > 0) { + List metrics = new ArrayList<>(); + PerformanceAnalyzerMetrics.metricQueue.drainTo(metrics); + LOG.info("Performance Analyzer no longer enabled. Drained the" + + "queue to remove stale data."); + } + return; + } + + LOG.info("Queue size {}", + PerformanceAnalyzerMetrics.metricQueue.size()); + + LOG.debug("Starting to purge the queue."); + List metrics = new ArrayList<>(); + PerformanceAnalyzerMetrics.metricQueue.drainTo(metrics); + LOG.debug("Queue draining successful."); + + long currentTimeMillis = System.currentTimeMillis(); + + // Calculate the timestamp on the file. For example, lets say the + // purging started at time 12.5 then all the events between 5-10 + // are written to a file with name 5. + long timeBucket = PerformanceAnalyzerMetrics.getTimeInterval( + currentTimeMillis, MetricsConfiguration.SAMPLING_INTERVAL) - + MetricsConfiguration.SAMPLING_INTERVAL; + + // When we are trying to collect the metrics for the 5th-10th second, + // but doing that in the 12.5th second, there is a chance that a + // collector ran in the 11th second and pushed the metrics in the + // queue. This thread, should be able to filter them and write them + // to their appropriate file, which should be 10 and not 5. + long nextTimeBucket = timeBucket + MetricsConfiguration.SAMPLING_INTERVAL; + + List currMetrics = new ArrayList<>(); + List nextMetrics = new ArrayList<>(); + + for (Event entry: metrics) { + if (entry.epoch == timeBucket) { + currMetrics.add(entry); + } else if (entry.epoch == nextTimeBucket) { + nextMetrics.add(entry); + } else { + LOG.info("UNEXPECTED entry ({}) with epoch '{}' arrived" + + " when the current bucket is '{}'", + entry.key, entry.epoch, timeBucket); + } + } + LOG.info("Curr Metrics size {}", currMetrics.size()); + LOG.info("Next Metrics size {}", nextMetrics.size()); + + LOG.debug("Start serializing and writing to file."); + writeAndRotate(currMetrics, timeBucket, currentTimeMillis); + if (!nextMetrics.isEmpty()) { + // The next bucket metrics don't need to be considered for + // rotation just yet. So, we just write them to the + // .tmp + eventLogFileHandler.writeTmpFile(nextMetrics, nextTimeBucket); + } + LOG.debug("Writing to disk complete."); + } + + private void writeAndRotate(final List currMetrics, + long currTimeBucket, + long currentTime) { + // Going by the continuing example, we will rotate the 5.tmp file to + // 5, which contains the metrics with epoch 5-10, whenever the purger + // runs after the 15th second. + if (lastTimeBucket != 0 && lastTimeBucket != currTimeBucket) { + eventLogFileHandler.renameFromTmp(lastTimeBucket); + } + // This appends the data to a file named .tmp + eventLogFileHandler.writeTmpFile(currMetrics, currTimeBucket); + lastTimeBucket = currTimeBucket; + } +} diff --git a/src/main/resources/plugin-descriptor.properties b/src/main/resources/plugin-descriptor.properties index c1a84627..e8115989 100644 --- a/src/main/resources/plugin-descriptor.properties +++ b/src/main/resources/plugin-descriptor.properties @@ -27,7 +27,7 @@ description=Performance Analyzer Plugin # # 'version': plugin's version -version=6.2.3 +version=7.1.1 # # 'name': the plugin name name=performance-analyzer @@ -42,4 +42,4 @@ classname=com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAna java.version=1.8 # # 'elasticsearch.version' version of elasticsearch compiled against -elasticsearch.version=6.2.3 +elasticsearch.version=7.1.1 diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/AbstractTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/AbstractTests.java index 71c08328..c7bf65e6 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/AbstractTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/AbstractTests.java @@ -32,6 +32,8 @@ import org.junit.Rule; import org.junit.rules.TemporaryFolder; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.Utils; + @Ignore public class AbstractTests { @@ -52,6 +54,7 @@ public AbstractTests() { @BeforeClass public static void setupLogging() { + Utils.configureMetrics(); ConfigurationBuilder configurationBuilder = ConfigurationBuilderFactory .newConfigurationBuilder(); configurationBuilder.setStatusLevel(Level.INFO); diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/CustomMetricsLocationTestBase.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/CustomMetricsLocationTestBase.java new file mode 100644 index 00000000..202e8cc1 --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/CustomMetricsLocationTestBase.java @@ -0,0 +1,25 @@ +package com.amazon.opendistro.elasticsearch.performanceanalyzer; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.junit.Before; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; + +public class CustomMetricsLocationTestBase { + + private static final Path METRICS_LOCATION = Paths.get("build/tmp/junit_metrics"); + + @Before + public void setUp() throws Exception { + if (!Files.exists(METRICS_LOCATION)) { + Files.createDirectories(METRICS_LOCATION.getParent()); + Files.createDirectory(METRICS_LOCATION); + } + + PluginSettings.instance().setMetricsLocation(METRICS_LOCATION + File.separator); + } +} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionListenerTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionListenerTests.java index 0c67ee6e..c3cdb071 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionListenerTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/action/PerformanceAnalyzerActionListenerTests.java @@ -15,11 +15,16 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.action; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import org.junit.Ignore; import org.junit.Test; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import static org.junit.Assert.assertEquals; -public class PerformanceAnalyzerActionListenerTests { +@Ignore +public class PerformanceAnalyzerActionListenerTests extends CustomMetricsLocationTestBase { @Test public void testHttpMetrics() { @@ -28,31 +33,31 @@ public void testHttpMetrics() { PerformanceAnalyzerActionListener performanceanalyzerActionListener = new PerformanceAnalyzerActionListener(); performanceanalyzerActionListener.saveMetricValues("XYZADFAS", startTimeInMills, "bulk", "bulkId", "start"); String fetchedValue = PerformanceAnalyzerMetrics.getMetric( - PerformanceAnalyzerMetrics.sDevShmLocation + PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/http/bulk/bulkId/start"); assertEquals("XYZADFAS", fetchedValue); String startMetricsValue = performanceanalyzerActionListener.generateStartMetrics(123, "val2", 0).toString(); performanceanalyzerActionListener.saveMetricValues(startMetricsValue, startTimeInMills, "search", "searchId1", "start"); fetchedValue = PerformanceAnalyzerMetrics.getMetric( - PerformanceAnalyzerMetrics.sDevShmLocation + PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/http/search/searchId1/start"); assertEquals(startMetricsValue, fetchedValue); String finishMetricsValue = performanceanalyzerActionListener.generateFinishMetrics(456, 200, "val4").toString(); performanceanalyzerActionListener.saveMetricValues(finishMetricsValue, startTimeInMills, "search", "searchId1", "finish"); fetchedValue = PerformanceAnalyzerMetrics.getMetric( - PerformanceAnalyzerMetrics.sDevShmLocation + PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/http/search/searchId1/finish"); assertEquals(finishMetricsValue, fetchedValue); performanceanalyzerActionListener.saveMetricValues(finishMetricsValue, startTimeInMills, "search", "searchId2", "finish"); fetchedValue = PerformanceAnalyzerMetrics.getMetric( - PerformanceAnalyzerMetrics.sDevShmLocation + PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/http/search/searchId2/finish"); assertEquals(finishMetricsValue, fetchedValue); - PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation + PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)); } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java index 573faef1..ed656418 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java @@ -15,21 +15,27 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; +import org.junit.Ignore; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -public class CircuitBreakerCollectorTests { +import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import static org.junit.Assert.assertEquals; +@Ignore +public class CircuitBreakerCollectorTests extends CustomMetricsLocationTestBase { @Test public void testCircuitBreakerMetrics() { + MetricsConfiguration.CONFIG_MAP.put(CircuitBreakerCollector.class, MetricsConfiguration.cdefault); System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); long startTimeInMills = 1153721339; CircuitBreakerCollector circuitBreakerCollector = new CircuitBreakerCollector(); circuitBreakerCollector.saveMetricValues("werjbdsiviewur", startTimeInMills); - String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PerformanceAnalyzerMetrics.sDevShmLocation + String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/circuit_breaker/"); - PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation + PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)); assertEquals("werjbdsiviewur", fetchedValue); diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetricsTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetricsTests.java index 1052a46b..1feefac7 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetricsTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/MasterServiceMetricsTests.java @@ -15,15 +15,26 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import org.junit.Ignore; import org.junit.Test; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceEventMetrics; -public class MasterServiceMetricsTests { +@Ignore +public class MasterServiceMetricsTests extends CustomMetricsLocationTestBase { @Test public void testMasterServiceMetrics() { + MetricsConfiguration.CONFIG_MAP.put(MasterServiceMetrics.class, new MetricsConfiguration.MetricConfig(1000, 0, 0)); + MetricsConfiguration.CONFIG_MAP.put(MasterServiceEventMetrics.class, new MetricsConfiguration.MetricConfig(1000, 0, 0)); System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); long startTimeInMills = 1353723339; @@ -31,9 +42,9 @@ public void testMasterServiceMetrics() { masterServiceMetrics.saveMetricValues("master_metrics_value", startTimeInMills, "current", "start"); - String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PerformanceAnalyzerMetrics.sDevShmLocation + + String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/pending_tasks/current/start/"); - PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation + PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)); assertEquals("master_metrics_value", fetchedValue); diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsMetricsCollectorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsMetricsCollectorTests.java index 2cb2f066..5d675f22 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsMetricsCollectorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/NodeStatsMetricsCollectorTests.java @@ -15,26 +15,34 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import org.junit.Ignore; import org.junit.Test; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -public class NodeStatsMetricsCollectorTests { +@Ignore +public class NodeStatsMetricsCollectorTests extends CustomMetricsLocationTestBase { @Test public void testNodeStatsMetrics() { System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); long startTimeInMills = 1253722339; + + MetricsConfiguration.CONFIG_MAP.put(NodeStatsMetricsCollector.class, MetricsConfiguration.cdefault); NodeStatsMetricsCollector nodeStatsMetricsCollector = new NodeStatsMetricsCollector(); nodeStatsMetricsCollector.saveMetricValues("89123.23", startTimeInMills, "NodesStatsIndex", "55"); String fetchedValue = PerformanceAnalyzerMetrics.getMetric( - PerformanceAnalyzerMetrics.sDevShmLocation + PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/indices/NodesStatsIndex/55/"); - PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation + PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)); assertEquals("89123.23", fetchedValue); diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ThreadPoolMetricsCollectorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ThreadPoolMetricsCollectorTests.java index 3026553b..3dc0cb3c 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ThreadPoolMetricsCollectorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/ThreadPoolMetricsCollectorTests.java @@ -15,24 +15,33 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import org.junit.Ignore; import org.junit.Test; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import static org.junit.Assert.assertEquals; -public class ThreadPoolMetricsCollectorTests { +@Ignore +public class ThreadPoolMetricsCollectorTests extends CustomMetricsLocationTestBase { @Test public void testThreadPoolMetrics() { System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); long startTimeInMills = 1453724339; - + + MetricsConfiguration.CONFIG_MAP.put(ThreadPoolMetricsCollector.class, MetricsConfiguration.cdefault); + ThreadPoolMetricsCollector threadPoolMetricsCollector = new ThreadPoolMetricsCollector(); threadPoolMetricsCollector.saveMetricValues("12321.5464", startTimeInMills); String fetchedValue = PerformanceAnalyzerMetrics.getMetric( - PerformanceAnalyzerMetrics.sDevShmLocation + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/thread_pool/"); - PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation + PluginSettings.instance().getMetricsLocation() + + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/thread_pool/"); + PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)); assertEquals("12321.5464", fetchedValue); @@ -50,5 +59,4 @@ public void testThreadPoolMetrics() { //- expecting exception...2 values passed; 0 expected } } - } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/GCMetricsTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/GCMetricsTests.java deleted file mode 100644 index 8c95edee..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/GCMetricsTests.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm; - -import org.junit.Test; - -public class GCMetricsTests { - public static void main(String[] args) throws Exception { - runOnce(); - } - - private static void runOnce() { - GCMetrics.runGCMetrics(); - GCMetrics.printGCMetrics(); - } - - //- to enhance - @Test - public void testMetrics() { - - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/HeapMetricsTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/HeapMetricsTests.java deleted file mode 100644 index 76bb6ce9..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/HeapMetricsTests.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm; - -import java.lang.management.MemoryUsage; -import java.util.Map; -import java.util.function.Supplier; - -import org.junit.Test; - -public class HeapMetricsTests { - public static void main(String[] args) throws Exception { - runOnce(); - } - - private static void runOnce() { - for (Map.Entry> entry : HeapMetrics.getMemoryUsageSuppliers().entrySet()) { - MemoryUsage memoryUsage = entry.getValue().get(); - System.out.println(entry.getKey() + "_committed:" + memoryUsage.getCommitted()); - System.out.println(entry.getKey() + "_init" + memoryUsage.getInit()); - System.out.println(entry.getKey() + "_max" + memoryUsage.getMax()); - System.out.println(entry.getKey() + "_used" + memoryUsage.getUsed()); - } - } - - //- to enhance - @Test - public void testMetrics() { - - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/ThreadListTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/ThreadListTests.java deleted file mode 100644 index 5bafdf05..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/jvm/ThreadListTests.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.jvm; - -import org.junit.Test; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.hwnet.NetworkInterface; -//import org.apache.logging.log4j.core.config.Configurator; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.os.OSGlobals; - -public class ThreadListTests { - //XXX: standalone test code - public static class HelloRunnable implements Runnable { - @Override - public void run() { - Thread.currentThread().setName("duMMy-thread"); - long i = 0; - while (true) { - synchronized (HelloRunnable.class) { - String.valueOf(i++); - } - } - } - } - public static void main(String[] args) throws Exception { - //Configurator.setAllLevels(LogManager.getRootLogger().getName(), Level.DEBUG); - (new Thread(new HelloRunnable())).start(); - (new Thread(new HelloRunnable())).start(); - runOnce(); - } - - private static void runOnce() throws InterruptedException { - String params[] = new String[0]; - while (true) { - ThreadList.runThreadDump(OSGlobals.getPid(), params); - ThreadList.LOGGER.info(ThreadList.getNativeTidMap().values()); - - /*GCMetrics.runOnce(); - HeapMetrics.runOnce(); - ThreadCPU.runOnce(); - ThreadDiskIO.runOnce(); - ThreadSched.runOnce(); - NetworkE2E.runOnce(); - Disks.runOnce();*/ - NetworkInterface.runOnce(); - - Thread.sleep(ThreadList.samplingInterval); - } - } - - //- to enhance - @Test - public void testMetrics() { - - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/listener/SearchListenerTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/listener/SearchListenerTests.java index ce24417f..738ce83b 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/listener/SearchListenerTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/listener/SearchListenerTests.java @@ -15,11 +15,16 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.listener; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import org.junit.Ignore; import org.junit.Test; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import static org.junit.Assert.assertEquals; -public class SearchListenerTests { +@Ignore +public class SearchListenerTests extends CustomMetricsLocationTestBase { @Test public void testShardSearchMetrics() { System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); @@ -27,31 +32,31 @@ public void testShardSearchMetrics() { PerformanceAnalyzerSearchListener performanceanalyzerSearchListener = new PerformanceAnalyzerSearchListener(); performanceanalyzerSearchListener.saveMetricValues("dewrjcve", startTimeInMills, "SearchThread", "shardquery", "ShardSearchID", "start"); - String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PerformanceAnalyzerMetrics.sDevShmLocation + + String fetchedValue = PerformanceAnalyzerMetrics.getMetric(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/SearchThread/shardquery/ShardSearchID/start"); assertEquals("dewrjcve", fetchedValue); String startMetricsValue = performanceanalyzerSearchListener.generateStartMetrics(100, "index1", 1).toString(); performanceanalyzerSearchListener.saveMetricValues(startMetricsValue, startTimeInMills, "SearchThread", "shardquery", "ShardSearchID1", "start"); - fetchedValue = PerformanceAnalyzerMetrics.getMetric(PerformanceAnalyzerMetrics.sDevShmLocation + + fetchedValue = PerformanceAnalyzerMetrics.getMetric(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/SearchThread/shardquery/ShardSearchID1/start"); assertEquals(startMetricsValue, fetchedValue); String finishMetricsValue = performanceanalyzerSearchListener.generateFinishMetrics(123, false, "index1", 10).toString(); performanceanalyzerSearchListener.saveMetricValues(finishMetricsValue, startTimeInMills, "SearchThread", "shardquery", "ShardSearchID1", "finish"); - fetchedValue = PerformanceAnalyzerMetrics.getMetric(PerformanceAnalyzerMetrics.sDevShmLocation + + fetchedValue = PerformanceAnalyzerMetrics.getMetric(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/SearchThread/shardquery/ShardSearchID1/finish"); assertEquals(finishMetricsValue, fetchedValue); performanceanalyzerSearchListener.saveMetricValues(finishMetricsValue, startTimeInMills, "SearchThread", "shardquery", "ShardSearchID2", "finish"); - fetchedValue = PerformanceAnalyzerMetrics.getMetric(PerformanceAnalyzerMetrics.sDevShmLocation + + fetchedValue = PerformanceAnalyzerMetrics.getMetric(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/SearchThread/shardquery/ShardSearchID2/finish"); assertEquals(finishMetricsValue, fetchedValue); - PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation + PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)); } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/PerformanceAnalyzerMetricsTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/PerformanceAnalyzerMetricsTests.java deleted file mode 100644 index 51a388fc..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/PerformanceAnalyzerMetricsTests.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics; - -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; -import org.powermock.modules.junit4.PowerMockRunner; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; -import static org.junit.Assert.assertEquals; - -@PowerMockIgnore({"org.apache.logging.log4j.*"}) -@RunWith(PowerMockRunner.class) -@PrepareForTest({PerformanceAnalyzerMetrics.class, PluginSettings.class}) -@SuppressStaticInitializationFor({"PluginSettings"}) -public class PerformanceAnalyzerMetricsTests { - - @Before - public void setUp() throws Exception { - PluginSettings config = Mockito.mock(PluginSettings.class); - Mockito.when(config.getMetricsLocation()).thenReturn("/dev/shm/performanceanalyzer"); - PowerMockito.mockStatic(PluginSettings.class); - PowerMockito.when(PluginSettings.instance()).thenReturn(config); - } - - @Test - public void testBasicMetric() { - System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); - PerformanceAnalyzerMetrics.emitMetric(PerformanceAnalyzerMetrics.sDevShmLocation + "/dir1/test1", "value1"); - assertEquals("value1", PerformanceAnalyzerMetrics.getMetric(PerformanceAnalyzerMetrics.sDevShmLocation + "/dir1/test1")); - - assertEquals("", PerformanceAnalyzerMetrics.getMetric(PerformanceAnalyzerMetrics.sDevShmLocation + "/dir1/test2")); - - PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation + "/dir1"); - } - - @Test - public void testGeneratePath() { - long startTimeInMillis = 1553725339; - String generatedPath = PerformanceAnalyzerMetrics.generatePath(startTimeInMillis, "dir1", "id", "dir2"); - String expectedPath = PerformanceAnalyzerMetrics.sDevShmLocation + - "/" + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMillis) + "/dir1/id/dir2"; - assertEquals(expectedPath, generatedPath); - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metricsdb/MetricsDBTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metricsdb/MetricsDBTests.java deleted file mode 100644 index dcf5a65e..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metricsdb/MetricsDBTests.java +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.jooq.BatchBindStep; -import org.jooq.Record; -import org.jooq.Result; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -@SuppressWarnings("serial") -public class MetricsDBTests { - private MetricsDB db; - - @Before - public void setUp() throws Exception { - System.setProperty("java.io.tmpdir", "/tmp"); - this.db = new MetricsDB(1553713380); - } - - public MetricsDBTests() throws ClassNotFoundException { - Class.forName("org.sqlite.JDBC"); - } - - @After - public void tearDown() throws Exception { - this.db.remove(); - } - - @Test - public void testTableCreation() throws Exception { - - List columns = Arrays.asList("shard", "index"); - db.createMetric(Metric.cpu(10D), columns); - - putCPUMetric(db, 10D, "1", "ac-test"); - putCPUMetric(db, 4D, "2", "ac-test"); - - Iterable res = db.queryMetric(Arrays.asList("cpu"), - Arrays.asList("sum"), - Arrays.asList("shard", "index")); - for(Record r: res) { - String s = r.get("shard").toString(); - assertTrue(s.equals("1")); - String index = r.get("index").toString(); - assertTrue(index.equals("ac-test")); - break; - } - - res = db.queryMetric(Arrays.asList("cpu"), - Arrays.asList("sum"), - Arrays.asList()); - for(Record r: res) { - Double sum = Double.parseDouble(r.get("cpu").toString()); - assertEquals(14D, sum.doubleValue(), 0); - } - db.close(); - } - - @Test - public void testTableNonexistent() throws Exception { - - List columns = Arrays.asList("shard", "index"); - db.createMetric(Metric.cpu(10D), columns); - - putCPUMetric(db, 10D, "1", "ac-test"); - putCPUMetric(db, 4D, "1", "ac-test"); - - Result res = db.queryMetric(Arrays.asList("pseudocpu"), - Arrays.asList("sum"), - Arrays.asList("shard", "index")); - assertNull(res); - - res = db.queryMetric(Arrays.asList("cpu", "pseudocpu"), - Arrays.asList("sum", "sum"), - Arrays.asList("shard", "index")); - - assertEquals(1, res.size()); - assertEquals(14D, Double.parseDouble(res.get(0).get("cpu").toString()), 0); - assertNull(res.get(0).get("pseudocpu")); - - db.close(); - } - - @Test - public void testMultiMetric() throws Exception { - List columns = Arrays.asList("shard", "index"); - db.createMetric(Metric.cpu(10D), columns); - db.createMetric(Metric.rss(10D), columns); - putCPUMetric(db, 10D, "1", "ac-test"); - putCPUMetric(db, 4D, "1", "ac-test"); - putRSSMetric(db, 1D, "1", "ac-test"); - putRSSMetric(db, 5D, "1", "ac-test"); - Iterable res = db.queryMetric(Arrays.asList("cpu", "rss"), - Arrays.asList("sum", "sum"), - Arrays.asList("shard", "index")); - for(Record r: res) { - Double cpu = Double.parseDouble(r.get("cpu").toString()); - Double rss = Double.parseDouble(r.get("rss").toString()); - assertEquals(14D, cpu.doubleValue(), 0); - assertEquals(6D, rss.doubleValue(), 0); - } - db.close(); - } - - - - @Test - public void testMultiMetricOuterJoin() throws Exception { - List columns = Arrays.asList("shard", "index"); - db.createMetric(Metric.cpu(10D), columns); - db.createMetric(Metric.rss(10D), columns); - putCPUMetric(db, 10D, "1", "ac-test"); - putCPUMetric(db, 20D, "2", "ac-test"); - putRSSMetric(db, 1D, "1", "ac-test"); - putRSSMetric(db, 3D, "3", "ac-test"); - Result res = db.queryMetric(Arrays.asList("cpu", "rss"), - Arrays.asList("sum", "sum"), - Arrays.asList("shard", "index")); - - assertEquals(3, res.size()); - boolean assert_cpu_only = false; - boolean assert_rss_only = false; - boolean assert_cpu_rss = false; - for (int i = 0; i < res.size(); i ++) { - Record r = res.get(i); - if (r.get("shard").toString().equals("1")) { - assert_cpu_rss = Double.parseDouble(r.get("cpu").toString()) == 10D && - Double.parseDouble(r.get("rss").toString()) == 1D; - } else if (r.get("shard").toString().equals("2")) { - assert_cpu_only = Double.parseDouble(r.get("cpu").toString()) == 20D; - } else if (r.get("shard").toString().equals("3")) { - assert_rss_only = Double.parseDouble(r.get("rss").toString()) == 3D; - } - } - assertTrue(assert_cpu_only); - assertTrue(assert_rss_only); - assertTrue(assert_cpu_rss); - db.close(); - } - - - - @Test - public void testMultiMetricNoDimension() throws Exception { - List columns = Arrays.asList("shard", "index"); - db.createMetric(Metric.cpu(10D), columns); - db.createMetric(Metric.rss(10D), columns); - putCPUMetric(db, 10D, "1", "ac-test"); - putCPUMetric(db, 30D, "2", "ac-test"); - putRSSMetric(db, 1D, "1", "ac-test"); - putRSSMetric(db, 3D, "3", "ac-test"); - Result res = db.queryMetric(Arrays.asList("cpu", "rss"), - Arrays.asList("avg", "sum"), - Arrays.asList()); - - assertEquals(1, res.size()); - - assertEquals(20D, Double.parseDouble(res.get(0).get("cpu").toString()), 0); - assertEquals(4D, Double.parseDouble(res.get(0).get("rss").toString()), 0); - db.close(); - } - - @Test - public void testGroupBy() throws Exception { - List columns = Arrays.asList("shard", "index"); - db.createMetric(Metric.cpu(10D), columns); - db.createMetric(Metric.rss(10D), columns); - putCPUMetric(db, 10D, "1", "ac-test"); - putCPUMetric(db, 4D, "1", "ac-test"); - putCPUMetric(db, 6D, "2", "ac-test"); - putCPUMetric(db, 8D, "1", "ac-test-2"); - putRSSMetric(db, 1D, "1", "ac-test"); - putRSSMetric(db, 5D, "1", "ac-test-2"); - putRSSMetric(db, 3D, "2", "ac-test"); - putRSSMetric(db, 3D, "2", "ac-test"); - Result res = db.queryMetric(Arrays.asList("cpu", "rss"), - Arrays.asList("sum", "sum"), - Arrays.asList("shard", "index")); - assertEquals(3, res.size()); - Double cpu = Double.parseDouble(res.get(0).get("cpu").toString()); - Double rss = Double.parseDouble(res.get(0).get("rss").toString()); - assertEquals(14D, cpu.doubleValue(), 0); - assertEquals(1D, rss.doubleValue(), 0); - cpu = Double.parseDouble(res.get(1).get("cpu").toString()); - rss = Double.parseDouble(res.get(1).get("rss").toString()); - assertEquals(8D, cpu.doubleValue(), 0); - assertEquals(5D, rss.doubleValue(), 0); - cpu = Double.parseDouble(res.get(2).get("cpu").toString()); - rss = Double.parseDouble(res.get(2).get("rss").toString()); - assertEquals(6D, cpu.doubleValue(), 0); - assertEquals(6D, rss.doubleValue(), 0); - } - - @Test - public void testAggAvg() throws Exception { - List columns = Arrays.asList("shard", "index"); - db.createMetric(Metric.cpu(10D), columns); - putCPUMetric(db, 10D, "1", "ac-test"); - putCPUMetric(db, 4D, "1", "ac-test"); - Result res = db.queryMetric(Arrays.asList("cpu"), - Arrays.asList("avg"), - Arrays.asList("shard", "index")); - Double cpu = Double.parseDouble(res.get(0).get("cpu").toString()); - assertEquals(7D, cpu, 0); - } - - private void putCPUMetric(MetricsDB db, Double val, String shard, String index) { - Dimensions dimensions = new Dimensions(); - dimensions.put("shard", shard); - dimensions.put("index", index); - db.putMetric(Metric.cpu(val), dimensions, 0); - } - - private void putRSSMetric(MetricsDB db, Double val, String shard, String index) { - Dimensions dimensions = new Dimensions(); - dimensions.put("shard", shard); - dimensions.put("index", index); - db.putMetric(Metric.rss(val), dimensions, 0); - } - - //@Test - public void perfTestSingleThread() throws Exception { - System.out.println("Batch Insert"); - long mCurrT = System.currentTimeMillis(); - //System.out.println("100000: "+runBatchTest(100000, "cpu")); - for (int i=0; i<5;i++) { - System.out.println("100000: cpu "+runBatchTest(100000,"cpu", db)); - System.out.println("100000: rss "+runBatchTest(100000,"rss", db)); - System.out.println("100000: paging "+runBatchTest(100000,"paging", db)); - } - long mFinalT = System.currentTimeMillis(); - db.commit(); - System.out.println("Total time taken: " + (mFinalT - mCurrT)); - //Thread.sleep(1000); - } - - //@Test - public void perfTest() throws Exception { - System.out.println("Batch Insert"); - long mCurrT = System.currentTimeMillis(); - //System.out.println("100000: "+runBatchTest(100000, "cpu")); - for (int i=0;i<5;i++) { - Thread t1 = new Thread(new Runnable() { - @Override - public void run() { - try { - System.out.println("100000: cpu "+runBatchTest(100000,"cpu", db)); - } catch(Exception e) { - System.out.println("Exception hit"); - System.out.println(e); - } - } - }); - - Thread t2 = new Thread(new Runnable() { - @Override - public void run() { - try { - System.out.println("100000: rss "+runBatchTest(100000,"rss", db)); - } catch(Exception e) { - System.out.println("Exception hit"); - System.out.println(e); - } - } - }); - - Thread t3 = new Thread(new Runnable() { - @Override - public void run() { - try { - System.out.println("100000: paging "+runBatchTest(100000,"paging", db)); - } catch(Exception e) { - System.out.println("Exception hit"); - System.out.println(e); - } - } - }); - - t2.start(); - t3.start(); - t1.start(); - t1.join(); - t2.join(); - t3.join(); - } - long mFinalT = System.currentTimeMillis(); - db.commit(); - System.out.println("Total time taken: " + (mFinalT - mCurrT)); - } - - private Long runBatchTest(int iterations, String metricColumn, MetricsDB mdb) throws Exception { - long mCurrT = System.currentTimeMillis(); - - List dims = new ArrayList() { { - this.add("shard"); - this.add("index"); - this.add("operation"); - this.add("role"); - } }; - mdb.createMetric(new Metric(metricColumn, 0d), dims); - BatchBindStep handle = mdb.startBatchPut(new Metric(metricColumn, 0d), dims); - - Dimensions dimensions = new Dimensions(); - for (int i=0; i Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.model; - -import org.junit.Test; - -import static org.junit.Assert.assertTrue; - - -public class ModelTests { - - @Test - public void testBasicMetric() { - MetricsModel metricAndDimensions = new MetricsModel(); - assertTrue(MetricsModel.ALL_METRICS.get("pseudocpu") == null); - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadCPUTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadCPUTests.java deleted file mode 100644 index 1dce9741..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadCPUTests.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.os; - -import org.junit.Test; - -public class ThreadCPUTests { - public static void main(String[] args) throws Exception { - runOnce(); - } - - private static void runOnce() { - ThreadCPU.INSTANCE.addSample(); - System.out.println("cpumap and pagemap:" + ThreadCPU.INSTANCE.getCPUPagingActivity().toString()); - } - - //- to enhance - @Test - public void testMetrics() { - - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadDiskIOTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadDiskIOTests.java deleted file mode 100644 index 3e79b0b0..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadDiskIOTests.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.os; - -import org.junit.Test; - -public class ThreadDiskIOTests { - public static void main(String[] args) throws Exception { - runOnce(); - } - - public static void runOnce() { - ThreadDiskIO.addSample(); - System.out.println(ThreadDiskIO.getIOUtilization().toString()); - } - - //- to enhance - @Test - public void testMetrics() { - - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadSchedTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadSchedTests.java deleted file mode 100644 index 551949c8..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/os/ThreadSchedTests.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.os; - -import org.junit.Test; - -public class ThreadSchedTests { - public static void main(String[] args) throws Exception { - runOnce(); - } - - public static void runOnce() { - ThreadSched.INSTANCE.addSample(); - System.out.println(ThreadSched.INSTANCE.getSchedLatency().toString()); - } - - //- to enhance - @Test - public void testMetrics() { - - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ClusterLevelMetricsReaderTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ClusterLevelMetricsReaderTests.java deleted file mode 100644 index e22a7410..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ClusterLevelMetricsReaderTests.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.sql.SQLException; - -//import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; -import org.powermock.modules.junit4.PowerMockRunner; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader.ClusterLevelMetricsReader.NodeDetails; - -@PowerMockIgnore({ "org.apache.logging.log4j.*" }) -@RunWith(PowerMockRunner.class) -@PrepareForTest({ PerformanceAnalyzerMetrics.class, PluginSettings.class }) -@SuppressStaticInitializationFor({ "PluginSettings" }) -public class ClusterLevelMetricsReaderTests extends AbstractReaderTests { - - public ClusterLevelMetricsReaderTests() throws SQLException, ClassNotFoundException { - super(); - // TODO Auto-generated constructor stub - } - - //@Test - public void testCollectNodeMetrics() throws Exception { - PluginSettings config = Mockito.mock(PluginSettings.class); - Mockito.when(config.getMetricsLocation()).thenReturn(rootLocation); - - PowerMockito.mockStatic(PluginSettings.class); - PowerMockito.when(PluginSettings.instance()).thenReturn(config); - - long currTimestamp = System.currentTimeMillis(); - long currTimeBucket = PerformanceAnalyzerMetrics.getTimeInterval(currTimestamp); - String currentTimeBucketStr = String.valueOf(currTimeBucket); - temporaryFolder.newFolder(currentTimeBucketStr); - File output = temporaryFolder.newFile(createRelativePath( - currentTimeBucketStr, PerformanceAnalyzerMetrics.sNodesPath)); - - String nodeId1 = "s7gDCVnCSiuBgHoYLji1gw"; - String address1 = "10.212.49.140"; - - String nodeId2 = "Zn1QcSUGT--DciD1Em5wRg"; - String address2 = "10.212.52.241"; - - write(output, false, - PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds(), - createNodeDetailsMetrics(nodeId1, address1), - createNodeDetailsMetrics(nodeId2, address2) - ); - -// setFinalStatic(PerformanceAnalyzerMetrics.class.getDeclaredField("sDevShmLocation"), -// rootLocation); - - ClusterLevelMetricsReader.collectNodeMetrics(currTimestamp); - - NodeDetails[] nodes = ClusterLevelMetricsReader.getNodes(); - - assertEquals(nodeId1, nodes[0].getId()); - assertEquals(address1, nodes[0].getHostAddress()); - - assertEquals(nodeId2, nodes[1].getId()); - assertEquals(address2, nodes[1].getHostAddress()); - } - - -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MasterEventMetricsSnapshotTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MasterEventMetricsSnapshotTests.java deleted file mode 100644 index 4855616e..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MasterEventMetricsSnapshotTests.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; -import org.jooq.BatchBindStep; -import org.jooq.Record; -import org.jooq.Result; -import org.junit.Before; -import org.junit.Test; - -import java.math.BigDecimal; -import java.sql.Connection; -import java.sql.DriverManager; - -import static org.junit.Assert.assertEquals; - - -public class MasterEventMetricsSnapshotTests { - - private static final String DB_URL = "jdbc:sqlite:"; - private Connection conn; - - @Before - public void setup() throws Exception { - Class.forName("org.sqlite.JDBC"); - System.setProperty("java.io.tmpdir", "/tmp"); - conn = DriverManager.getConnection(DB_URL); - } - - @Test - public void testStartEventOnly() { - MasterEventMetricsSnapshot masterEventMetricsSnapshot = new MasterEventMetricsSnapshot(conn, 1535065195000L); - BatchBindStep handle = masterEventMetricsSnapshot.startBatchPut(); - - handle.bind("111","1","urgent","create-index","metadata",12,1535065195001L,null); - handle.execute(); - Result rt = masterEventMetricsSnapshot.fetchQueueAndRunTime(); - - assertEquals(1, rt.size()); - assertEquals(4999L, - ((BigDecimal)(rt.get(0).get("sum_" + AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString()))).longValue()); - assertEquals("urgent", - rt.get(0).get(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString())); - assertEquals("create-index", - rt.get(0).get(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString())); - assertEquals("metadata", - rt.get(0).get(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString())); - assertEquals(12L, - ((BigDecimal)(rt.get(0).get("sum_" + AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString()))).longValue()); - } - - @Test - public void testStartAndEndEvents() { - MasterEventMetricsSnapshot masterEventMetricsSnapshot = new MasterEventMetricsSnapshot(conn, 1535065195000L); - BatchBindStep handle = masterEventMetricsSnapshot.startBatchPut(); - - handle.bind("111","1","urgent","create-index","metadata",12,1535065195001L,null); - handle.bind("111","1",null,null,null,12, null, 1535065195005L); - handle.execute(); - Result rt = masterEventMetricsSnapshot.fetchQueueAndRunTime(); - - assertEquals(1, rt.size()); - assertEquals(4L, - ((BigDecimal)(rt.get(0).get("sum_" + AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString()))).longValue()); - assertEquals("urgent", - rt.get(0).get(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString())); - assertEquals("create-index", - rt.get(0).get(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString())); - assertEquals("metadata", - rt.get(0).get(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString())); - assertEquals(12L, - ((BigDecimal)(rt.get(0).get("sum_" + AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString()))).longValue()); - } - - @Test - public void testMultipleInsertOrderStartAndEndEvents() { - MasterEventMetricsSnapshot masterEventMetricsSnapshot = new MasterEventMetricsSnapshot(conn, 1535065195000L); - BatchBindStep handle = masterEventMetricsSnapshot.startBatchPut(); - - handle.bind("111","1","urgent","create-index","metadata",12,1535065195001L,null); - handle.bind("111","1",null,null,null,12, null, 1535065195005L); - handle.bind("111","2","high","remapping","metadata2",2,1535065195007L,null); - handle.execute(); - - Result rt = masterEventMetricsSnapshot.fetchQueueAndRunTime(); - - assertEquals(2, rt.size()); - assertEquals(4L, - ((BigDecimal)(rt.get(0).get("sum_" + AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString()))).longValue()); - assertEquals("urgent", - rt.get(0).get(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString())); - assertEquals("create-index", - rt.get(0).get(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString())); - assertEquals("metadata", - rt.get(0).get(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString())); - assertEquals(12L, - ((BigDecimal)(rt.get(0).get("sum_" + AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString()))).longValue()); - - assertEquals(4993L, - ((BigDecimal)(rt.get(1).get("sum_" + AllMetrics.MasterMetricDimensions.MASTER_TASK_RUN_TIME.toString()))).longValue()); - assertEquals("high", - rt.get(1).get(AllMetrics.MasterMetricDimensions.MASTER_TASK_PRIORITY.toString())); - assertEquals("remapping", - rt.get(1).get(AllMetrics.MasterMetricDimensions.MASTER_TASK_TYPE.toString())); - assertEquals("metadata2", - rt.get(1).get(AllMetrics.MasterMetricDimensions.MASTER_TASK_METADATA.toString())); - assertEquals(2L, - ((BigDecimal)(rt.get(1).get("sum_" + AllMetrics.MasterMetricDimensions.MASTER_TASK_QUEUE_TIME.toString()))).longValue()); - - } - - @Test - public void testRollOver() { - MasterEventMetricsSnapshot masterEventMetricsSnapshotPre = new MasterEventMetricsSnapshot(conn, 1535065195000L); - BatchBindStep handle = masterEventMetricsSnapshotPre.startBatchPut(); - - handle.bind("111","1","urgent","create-index","metadata",12,1535065195001L,null); - handle.execute(); - - MasterEventMetricsSnapshot masterEventMetricsSnapshotCurrent = new MasterEventMetricsSnapshot(conn, 1535065200000L); - Result rt = masterEventMetricsSnapshotCurrent.fetchAll(); - assertEquals(0, rt.size()); - - masterEventMetricsSnapshotCurrent.rolloverInflightRequests(masterEventMetricsSnapshotPre); - - Result rt2 = masterEventMetricsSnapshotCurrent.fetchAll(); - assertEquals(1, rt2.size()); - } - - @Test - public void testNotRollOverExpired() { - MasterEventMetricsSnapshot masterEventMetricsSnapshotPre = new MasterEventMetricsSnapshot(conn, 1535065195000L); - BatchBindStep handle = masterEventMetricsSnapshotPre.startBatchPut(); - - handle.bind("111","1","urgent","create-index","metadata",12,1435065195001L,null); - handle.execute(); - - MasterEventMetricsSnapshot masterEventMetricsSnapshotCurrent = new MasterEventMetricsSnapshot(conn, 1535065200000L); - Result rt = masterEventMetricsSnapshotCurrent.fetchAll(); - assertEquals(0, rt.size()); - - masterEventMetricsSnapshotCurrent.rolloverInflightRequests(masterEventMetricsSnapshotPre); - - Result rt2 = masterEventMetricsSnapshotCurrent.fetchAll(); - assertEquals(0, rt2.size()); - } - -} - diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MemoryDBSnapshotTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MemoryDBSnapshotTests.java deleted file mode 100644 index 2faf1b9d..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MemoryDBSnapshotTests.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import static org.hamcrest.Matchers.closeTo; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.impl.DSL; -import org.junit.Test; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CircuitBreakerDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CircuitBreakerValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricName; - -public class MemoryDBSnapshotTests extends AbstractReaderTests { - - private Field[] fields; - - //- May change across versions, based on new/removed circuit breakers - - enum CircuitBreakerType { - request, fielddata, in_flight_requests, accounting, parent; - } - - @SuppressWarnings("unchecked") - public MemoryDBSnapshotTests() throws SQLException, ClassNotFoundException { - super(); - - fields = new Field[3]; - - fields[0] = DSL.field(CircuitBreakerValue.CB_ESTIMATED_SIZE.toString(), - Double.class); - fields[1] = - DSL.field(CircuitBreakerValue.CB_CONFIGURED_SIZE.toString(), Double.class); - fields[2] = DSL.field(CircuitBreakerValue.CB_TRIPPED_EVENTS.toString(), - Double.class); - } - - @Test - public void testCreateMemoryDBSnapshot() throws Exception { - Connection conn = DriverManager.getConnection(DB_URL); - - long windowEndTime = 1L; - - MemoryDBSnapshot circuitBreakerValuesSnap = new MemoryDBSnapshot(conn, - MetricName.CIRCUIT_BREAKER, - windowEndTime); - - assertEquals( - MetricName.CIRCUIT_BREAKER.toString() - + windowEndTime, - circuitBreakerValuesSnap.getTableName()); - - long lastUpdatedTime = 2L; - circuitBreakerValuesSnap.setLastUpdatedTime(lastUpdatedTime); - assertTrue(lastUpdatedTime == circuitBreakerValuesSnap - .getLastUpdatedTime()); - - // should cause no exception - circuitBreakerValuesSnap.insertMultiRows(null); - - assertTrue (0 == circuitBreakerValuesSnap.fetchAll().size()); - - - Object[][] values = { - { CircuitBreakerType.request.toString(), 0, 0d, - 19607637196d }, - { CircuitBreakerType.fielddata.toString(), 0, 0, - 19607637196d }, - { CircuitBreakerType.in_flight_requests.toString(), 0, - 0, 32679395328d }, - { CircuitBreakerType.accounting.toString(), 0, 0, - 32679395328d }, - { CircuitBreakerType.parent.toString(), 0, 0, - 22875576729d } }; - - circuitBreakerValuesSnap.insertMultiRows(values); - - assertTrue (5 == circuitBreakerValuesSnap.fetchAll().size()); - - // no need to commit as database is in auto-commit mode - - - Result resultRecord = circuitBreakerValuesSnap.fetchMetric( - getDimensionEqCondition( - CircuitBreakerDimension.CB_TYPE, - CircuitBreakerType.request.toString()), - fields); - - assertTrue(1 == resultRecord.size()); - - Record r = resultRecord.get(0); - Double estimated = r.get(fields[0]); - assertThat(estimated, closeTo(0, 0.1)); - - Double tripped = r.get(fields[1]); - assertThat(tripped, closeTo(19607637196d, 0.1)); - - Double limit = r.get(fields[2]); - assertThat(limit, closeTo(0, 0.1)); - - // The 2nd remove should have no effect since the db table has already - // been deleted - for (int i = 0; i < 2; i++) { - circuitBreakerValuesSnap.remove(); - assertTrue(!circuitBreakerValuesSnap.dbTableExists()); - } - } - - @Test - public void testAlignWindow() throws Exception { - // time line - // writer writes at 2000l - // reader reads at 6000l - // writer writes at 7000l - // reader reads at 11000l - // writer writes at 12000l - - MemoryDBSnapshot circuitBreakerValuesSnap1 = new MemoryDBSnapshot(conn, - MetricName.CIRCUIT_BREAKER, 6000L); - circuitBreakerValuesSnap1.setLastUpdatedTime(2000L); - Object[][] values1 = { - {CircuitBreakerType.fielddata.toString(), - 0, 1, 19607637196d } }; - circuitBreakerValuesSnap1.insertMultiRows(values1); - - MemoryDBSnapshot circuitBreakerValuesSnap2 = new MemoryDBSnapshot(conn, - MetricName.CIRCUIT_BREAKER, 11000L); - circuitBreakerValuesSnap2.setLastUpdatedTime(7000L); - Object[][] values2 = { - { CircuitBreakerType.fielddata.toString(), - 0, 2, 19607637196d } }; - circuitBreakerValuesSnap1.insertMultiRows(values2); - - // The 3rd parameter is windowEndTime. - // So current time is 11000. But we use PerformanceAnalyzerMetrics.getTimeInterval to - // compute the aligned reader window time: 10000. - // So our aligned window time is [5000,10000]. - MemoryDBSnapshot circuitFinal = new MemoryDBSnapshot(conn, - MetricName.CIRCUIT_BREAKER, 10000L, true); - circuitFinal.alignWindow(circuitBreakerValuesSnap1, - circuitBreakerValuesSnap2, 7000L, 5000L, 10000L); - - Result res = circuitFinal.fetchMetric( - getDimensionEqCondition(CircuitBreakerDimension.CB_TYPE, - CircuitBreakerType.fielddata.toString()), - fields); - Double estimated = Double - .parseDouble(res.get(0).get(fields[0]).toString()); - - assertEquals(estimated, 0, 0.001); - - Double tripped = Double.parseDouble( - res.get(0).get(fields[2]).toString()); - assertEquals(tripped, 1.5, 0.001); - - Double limit = Double.parseDouble( - res.get(0).get(fields[1]).toString()); - assertEquals(limit, 19607637196d, 0.001); - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesConfigTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesConfigTests.java deleted file mode 100644 index c93e6b6d..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesConfigTests.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.sql.SQLException; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.jooq.Condition; -import org.jooq.Field; -import org.junit.Test; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricName; - -public class MetricPropertiesConfigTests extends AbstractReaderTests { - - public MetricPropertiesConfigTests() throws SQLException, ClassNotFoundException { - super(); - } - - /** - * Test table names used for metadata table in disk db are unique - */ - @Test - public void testUniqueTableNames() { - Set seen = new HashSet<>(); - for (Map.Entry entry : MetricPropertiesConfig - .getInstance().getMetricName2Property().entrySet()) { - MetricProperties property = entry.getValue(); - List metadataTableNames = property.getMetadataTableNames(); - for (String name : metadataTableNames) { - assertTrue(String.format( - "Metric %s has duplicate metadata table name %s", - entry.getKey(), name), !seen.contains(name)); - seen.add(name); - } - } - } - - @Test - public void testConsistentAcrossMaps() { - for (Map.Entry entry : MetricPropertiesConfig - .getInstance().getMetricName2Property().entrySet()) { - MetricProperties property = entry.getValue(); - - List metadataTableNames = property.getMetadataTableNames(); - Map>> groupByFields = property - .getTableGroupByFieldsMap(); - Map>> selectFields = property - .getTableSelectMap(); - Map whereClauses = property - .getTableWhereClauseMap(); - - assertEquals(metadataTableNames.size(), groupByFields.size()); - assertEquals(metadataTableNames.size(), selectFields.size()); - assertEquals(metadataTableNames.size(), whereClauses.size()); - - for (String tableName : metadataTableNames) { - assertTrue(groupByFields.containsKey(tableName)); - assertTrue(selectFields.containsKey(tableName)); - assertTrue(whereClauses.containsKey(tableName)); - - List> currGroupByFields = groupByFields - .get(tableName); - List> currSelectFields = selectFields.get(tableName); - for (Field field : currGroupByFields) { - assertTrue(currSelectFields.contains(field)); - } - } - } - } - - /** - * Test if we have configuration for each MetricName - */ - @Test - public void testMetricNameConsistent() { - for (MetricName name : MetricName.values()) { - assertTrue(String.format("Missing %s", name), MetricPropertiesConfig - .getInstance().getMetricName2Property().containsKey(name)); - } - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesTests.java index 158588d1..c83b4b99 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesTests.java @@ -16,9 +16,6 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import java.io.File; import java.io.IOException; import java.sql.SQLException; @@ -27,9 +24,11 @@ import org.jooq.Record; import org.jooq.Result; import org.jooq.impl.DSL; +import org.junit.Ignore; import org.junit.Test; import org.mockito.Mockito; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.GCType; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HeapDimension; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HeapValue; @@ -37,7 +36,10 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardStatsDerivedDimension; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardStatsValue; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +@Ignore public class MetricPropertiesTests extends AbstractReaderTests { public MetricPropertiesTests() throws SQLException, ClassNotFoundException { @@ -377,7 +379,8 @@ public void testBadPathPattern() throws Exception { @Test public void testDefaultRootLocation() { - assertEquals(PerformanceAnalyzerMetrics.sDevShmLocation, + assertEquals( + PluginSettings.instance().getMetricsLocation(), MetricPropertiesConfig .createFileHandler(PerformanceAnalyzerMetrics.sCircuitBreakerPath) .getRootLocation()); diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricsEmitterTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricsEmitterTests.java deleted file mode 100644 index be4c6b86..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricsEmitterTests.java +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.closeTo; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -import org.jooq.BatchBindStep; -import org.jooq.DSLContext; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.SQLDialect; -import org.jooq.impl.DSL; -import org.junit.Test; -//import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; -//import org.powermock.core.classloader.annotations.PowerMockIgnore; -//import org.powermock.core.classloader.annotations.PrepareForTest; -//import org.powermock.modules.junit4.PowerMockRunner; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.TroubleshootingConfig; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CommonMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HttpMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricName; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.OSMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardBulkMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardOperationMetric; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.TCPDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.TCPValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.Dimensions; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; - -//@PowerMockIgnore({ "org.apache.logging.log4j.*" }) -//@RunWith(PowerMockRunner.class) -//@PrepareForTest({ TroubleshootingConfig.class }) -public class MetricsEmitterTests extends AbstractReaderTests { - public MetricsEmitterTests() throws SQLException, ClassNotFoundException { - super(); - // TODO Auto-generated constructor stub - } - - private static final String DB_URL = "jdbc:sqlite:"; - - @Test - public void testMetricsEmitter() throws Exception { - // - Connection conn = DriverManager.getConnection(DB_URL); - ShardRequestMetricsSnapshot rqMetricsSnap = new ShardRequestMetricsSnapshot(conn, 1535065195000L); - Map dimensions = new HashMap<>(); - dimensions.put(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString(), "ac-test"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString(), "1"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.OPERATION.toString(), "shardBulk"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.SHARD_ROLE.toString(), "primary"); - dimensions.put("tid", "1"); - dimensions.put("rid", "1"); - rqMetricsSnap.putStartMetric(1535065196120L, dimensions); - rqMetricsSnap.putEndMetric(1535065196323L, dimensions); - dimensions.put("rid", "2"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.OPERATION.toString(), "shardSearch"); - rqMetricsSnap.putStartMetric(1535065197323L, dimensions); - dimensions.put("rid", "3"); - dimensions.put("tid", "2"); - rqMetricsSnap.putStartMetric(1535065198323L, dimensions); - rqMetricsSnap.putEndMetric(1535065199923L, dimensions); - Result res = rqMetricsSnap.fetchThreadUtilizationRatio(); - Float tUtil = Float.parseFloat(res.get(0).get("tUtil").toString()); - assertEquals(0.07048611f, tUtil.floatValue(), 0); - - OSMetricsSnapshot osMetricsSnap = new OSMetricsSnapshot(conn, 1L); - //Create OSMetricsSnapshot - Map metrics = new HashMap<>(); - Map osDim = new HashMap<>(); - osDim.put("tid", "1"); - osDim.put("tName", "elasticsearch[E-C7clp][search][T#1]"); - metrics.put(OSMetrics.CPU_UTILIZATION.toString(), 2.3333d); - metrics.put(OSMetrics.PAGING_RSS.toString(), 3.63d); - osMetricsSnap.putMetric(metrics, osDim, 1L); - osDim.put("tid", "2"); - osDim.put("tName", "elasticsearch[E-C7clp][bulk][T#2]"); - metrics.put(OSMetrics.CPU_UTILIZATION.toString(), 3.3333d); - metrics.put(OSMetrics.PAGING_RSS.toString(), 1.63d); - osMetricsSnap.putMetric(metrics, osDim, 1L); - osDim.put("tid", "3"); - osDim.put("tName", "GC"); - metrics.put(OSMetrics.CPU_UTILIZATION.toString(), 3.3333d); - metrics.put(OSMetrics.PAGING_RSS.toString(), 1.63d); - osMetricsSnap.putMetric(metrics, osDim, 1L); - - DSLContext create = DSL.using(conn, SQLDialect.SQLITE); - MetricsDB db = new MetricsDB(1553713402); - MetricsEmitter.emitAggregatedOSMetrics(create, db, osMetricsSnap, rqMetricsSnap); - res = db.queryMetric(Arrays.asList(OSMetrics.PAGING_RSS.toString(), OSMetrics.CPU_UTILIZATION.toString()), - Arrays.asList("sum", "sum"), - Arrays.asList(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString(), - ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString(), - ShardRequestMetricsSnapshot.Fields.OPERATION.toString())); - - Double cpu = Double.parseDouble(res.get(0).get(OSMetrics.CPU_UTILIZATION.toString()).toString()); - db.remove(); - assertEquals(0.164465243055556d, cpu.doubleValue(), 0); - } - - - //@Test(expected = Exception.class) - public void testMetricsEmitterInvalidData() throws Exception { - // - PowerMockito.mockStatic(TroubleshootingConfig.class); - PowerMockito.when(TroubleshootingConfig.getEnableDevAssert()).thenReturn(true); - - Connection conn = DriverManager.getConnection(DB_URL); - ShardRequestMetricsSnapshot rqMetricsSnap = new ShardRequestMetricsSnapshot(conn, 1535065195000L); - Map dimensions = new HashMap<>(); - dimensions.put(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString(), "ac-test"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString(), "1"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.OPERATION.toString(), "shardBulk"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.SHARD_ROLE.toString(), "primary"); - dimensions.put("tid", "1"); - dimensions.put("rid", "1"); - rqMetricsSnap.putStartMetric(1535065196120L, dimensions); - rqMetricsSnap.putEndMetric(1535065196323L, dimensions); - dimensions.put("rid", "2"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.OPERATION.toString(), "shardSearch"); - rqMetricsSnap.putStartMetric(1535065197323L, dimensions); - dimensions.put("rid", "3"); - dimensions.put("tid", "2"); - rqMetricsSnap.putStartMetric(1535065198323L, dimensions); - rqMetricsSnap.putEndMetric(1535065199923L, dimensions); - Result res = rqMetricsSnap.fetchThreadUtilizationRatio(); - Float tUtil = Float.parseFloat(res.get(0).get("tUtil").toString()); - assertEquals(0.07048611f, tUtil.floatValue(), 0); - - OSMetricsSnapshot osMetricsSnap = new OSMetricsSnapshot(conn, 1L); - //Create OSMetricsSnapshot - Map metrics = new HashMap<>(); - Map osDim = new HashMap<>(); - osDim.put("tid", "1"); - osDim.put("tName", "elasticsearch[E-C7clp][search][T#1]"); - metrics.put(OSMetrics.CPU_UTILIZATION.toString(), 2.3333d); - metrics.put(OSMetrics.PAGING_RSS.toString(), 3.63d); - osMetricsSnap.putMetric(metrics, osDim, 1L); - osDim.put("tid", "2"); - osDim.put("tName", "GC thread"); - metrics.put(OSMetrics.CPU_UTILIZATION.toString(), 3.3333d); - metrics.put(OSMetrics.PAGING_RSS.toString(), 1.63d); - osMetricsSnap.putMetric(metrics, osDim, 1L); - osDim.put("tid", "3"); - osDim.put("tName", "GC"); - metrics.put(OSMetrics.CPU_UTILIZATION.toString(), 3.3333d); - metrics.put(OSMetrics.PAGING_RSS.toString(), 1.63d); - osMetricsSnap.putMetric(metrics, osDim, 1L); - - DSLContext create = DSL.using(conn, SQLDialect.SQLITE); - MetricsDB db = new MetricsDB(1553713410); - MetricsEmitter.emitAggregatedOSMetrics(create, db, osMetricsSnap, rqMetricsSnap); - res = db.queryMetric(Arrays.asList(OSMetrics.PAGING_RSS.toString(), OSMetrics.CPU_UTILIZATION.toString()), - Arrays.asList("sum", "sum"), - Arrays.asList(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString(), - ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString(), - ShardRequestMetricsSnapshot.Fields.OPERATION.toString())); - db.remove(); - } - - @Test - public void testHttpMetricsEmitter() throws Exception { - Connection conn = DriverManager.getConnection(DB_URL); - HttpRequestMetricsSnapshot rqMetricsSnap = new HttpRequestMetricsSnapshot(conn, 1L); - Map dimensions = new HashMap<>(); - dimensions.put(HttpRequestMetricsSnapshot.Fields.OPERATION.toString(), "search"); - dimensions.put(HttpRequestMetricsSnapshot.Fields.HTTP_RESP_CODE.toString(), "200"); - dimensions.put(HttpRequestMetricsSnapshot.Fields.INDICES.toString(), ""); - dimensions.put(HttpRequestMetricsSnapshot.Fields.EXCEPTION.toString(), ""); - dimensions.put("rid", "1"); - rqMetricsSnap.putStartMetric(12345L, 0L, dimensions); - rqMetricsSnap.putEndMetric(33325L, dimensions); - dimensions.put("rid", "2"); - dimensions.put(HttpRequestMetricsSnapshot.Fields.OPERATION.toString(), "search"); - rqMetricsSnap.putStartMetric(22245L,0L,dimensions); - dimensions.put("rid", "3"); - rqMetricsSnap.putStartMetric(10000L,0L,dimensions); - rqMetricsSnap.putEndMetric(30000L, dimensions); - - DSLContext create = DSL.using(conn, SQLDialect.SQLITE); - MetricsDB db = new MetricsDB(1553713438); - MetricsEmitter.emitHttpMetrics(create, db, rqMetricsSnap); - Result res = db.queryMetric(Arrays.asList(CommonMetric.LATENCY.toString(), - HttpMetric.HTTP_TOTAL_REQUESTS.toString()), - Arrays.asList("avg", "sum"), - Arrays.asList(HttpRequestMetricsSnapshot.Fields.OPERATION.toString())); - - Float latency = Float.parseFloat(res.get(0).get(CommonMetric.LATENCY.toString()).toString()); - db.remove(); - assertEquals(20490.0f, latency.floatValue(), 0); - } - - @Test - public void testWorkloadMetricsEmitter() throws Exception { - Connection conn = DriverManager.getConnection(DB_URL); - ShardRequestMetricsSnapshot rqMetricsSnap = new ShardRequestMetricsSnapshot(conn, 1535065195000L); - BatchBindStep handle = rqMetricsSnap.startBatchPut(); - handle.bind("shardId", "indexName", "1", "threadId", "operation", "primary", 1535065195000L, null, 10); - handle.bind("shardId", "indexName", "1", "threadId", "operation", "primary", null, 1535065196000L, null); - handle.bind("shardId", "indexName", "2", "threadId", "operation", "primary", 1535065197000L, null, 10); - handle.bind("shardId", "indexName", "2", "threadId", "operation", "primary", null, 1535065198000L, null); - handle.execute(); - - System.out.println(rqMetricsSnap.fetchAll()); - System.out.println(rqMetricsSnap.fetchLatencyByOp()); - - DSLContext create = DSL.using(conn, SQLDialect.SQLITE); - MetricsDB db = new MetricsDB(1553713445); - MetricsEmitter.emitWorkloadMetrics(create, db, rqMetricsSnap); - Result res = db.queryMetric(Arrays.asList(ShardBulkMetric.DOC_COUNT.toString(), - ShardOperationMetric.SHARD_OP_COUNT.toString()), - Arrays.asList("sum", "sum"), - Arrays.asList(HttpRequestMetricsSnapshot.Fields.OPERATION.toString())); - - Double bulkDocs = Double.parseDouble(res.get(0).get(ShardBulkMetric.DOC_COUNT.toString()).toString()); - Double shardOps = Double.parseDouble(res.get(0).get(ShardOperationMetric.SHARD_OP_COUNT.toString()).toString()); - db.remove(); - assertEquals(20.0d, bulkDocs.doubleValue(), 0); - assertEquals(2d, shardOps.doubleValue(), 0); - } - - @Test - public void testWorkloadMetricsEmitterDoNothing() throws Exception { - Connection conn = DriverManager.getConnection(DB_URL); - ShardRequestMetricsSnapshot rqMetricsSnap = new ShardRequestMetricsSnapshot(conn, 1535065195000L); - DSLContext create = DSL.using(conn, SQLDialect.SQLITE); - MetricsDB db = new MetricsDB(1553713492); - MetricsEmitter.emitWorkloadMetrics(create, db, rqMetricsSnap); - System.out.println(rqMetricsSnap.fetchAll()); - db.remove(); - assertEquals(0, rqMetricsSnap.fetchAll().size()); - } - - @Test - public void testExtractor() { - String check = "abc: 2\nbbc:\ncbc:21\n"; - assertEquals(" 2", PerformanceAnalyzerMetrics.extractMetricValue(check, "abc")); - assertEquals("", PerformanceAnalyzerMetrics.extractMetricValue(check, "bbc")); - assertEquals("21", PerformanceAnalyzerMetrics.extractMetricValue(check, "cbc")); - } - - @Test - public void testThreadNameCategorization() { - Dimensions dimensions = new Dimensions(); - assertEquals("GC", MetricsEmitter.categorizeThreadName("Gang worker#0 (Parallel GC Threads)", dimensions)); - assertEquals(null , MetricsEmitter.categorizeThreadName("elasticsearch[I9AByra][search][T#4]", dimensions)); - assertEquals("refresh", MetricsEmitter.categorizeThreadName("elasticsearch[I9AByra][refresh][T#1]", dimensions)); - assertEquals("merge", MetricsEmitter.categorizeThreadName("elasticsearch[I9AByra][[nyc_taxis][1]: Lucene Merge", dimensions)); - assertEquals("management", MetricsEmitter.categorizeThreadName("elasticsearch[I9AByra][management]", dimensions)); - assertEquals(null, MetricsEmitter.categorizeThreadName("elasticsearch[I9AByra][search]", dimensions)); - assertEquals(null, MetricsEmitter.categorizeThreadName("elasticsearch[I9AByra][bulk]", dimensions)); - assertEquals("other", MetricsEmitter.categorizeThreadName("Top thread random", dimensions)); - } - - @Test - public void testEmitNodeMetrics() throws Exception { - Connection conn = DriverManager.getConnection(DB_URL); - - MemoryDBSnapshot tcpSnap = new MemoryDBSnapshot(conn, - MetricName.TCP_METRICS, - 5001L); - - long lastUpdatedTime = 2000L; - tcpSnap.setLastUpdatedTime(lastUpdatedTime); - - Object[][] values = { - { "0000000000000000FFFF0000E03DD40A", 24, 0, 0, 0, 7, 1 }, - { "0000000000000000FFFF00006733D40A", 23, 0, 0, 0, 6, 1 }, - { "0000000000000000FFFF00000100007F", 24, 0, 0, 0, 10,-1 }, - { "0000000000000000FFFF00005432D40A", 23, 0, 0, 0, 8,5 }, - { "00000000000000000000000000000000", 4, 0, 0, 0, 10, 0 }, - { "0000000000000000FFFF0000F134D40A", 23, 0, 0, 0, 8, 0}}; - - tcpSnap.insertMultiRows(values); - - DSLContext create = DSL.using(conn, SQLDialect.SQLITE); - MetricsDB db = new MetricsDB(1553713499); - MetricsEmitter.emitNodeMetrics(create, db, tcpSnap); - Result res = db.queryMetric( - Arrays.asList(TCPValue.Net_TCP_NUM_FLOWS.toString(), - TCPValue.Net_TCP_SSTHRESH.toString()), - Arrays.asList("sum", "avg"), - Arrays.asList(TCPDimension.DEST_ADDR.toString())); - - assertTrue(6 == res.size()); - - for (int i = 0; i < 6; i++) { - Record record0 = res.get(i); - Double numFlows = Double.parseDouble( - record0.get(TCPValue.Net_TCP_NUM_FLOWS.toString()).toString()); - - assertThat(numFlows.doubleValue(), anyOf(closeTo(24, 0.001), - closeTo(23, 0.001), closeTo(4, 0.001))); - - Double ssThresh = Double.parseDouble( - record0.get(TCPValue.Net_TCP_SSTHRESH.toString()).toString()); - - assertThat(ssThresh.doubleValue(), anyOf(closeTo(1, 0.001), - closeTo(-1, 0.001), closeTo(5, 0.001), closeTo(0, 0.001))); - - } - db.remove(); - - } -} - - diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricsParserTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricsParserTest.java deleted file mode 100644 index 09fcd75d..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricsParserTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.CopyTestResource; -import org.junit.Test; - -import java.io.File; - -public class MetricsParserTest { - private static final String DB_URL = "jdbc:sqlite:"; - - @Test - public void testMetricsParser() throws Exception { - ReaderMetricsProcessor mp = new ReaderMetricsProcessor("build/private/test_resources/dev/shm"); - MetricsParser parser = new MetricsParser(); - OSMetricsSnapshot osMetricsSnap = new OSMetricsSnapshot(mp.getConnection(), 1L); - ShardRequestMetricsSnapshot rqSnap = new ShardRequestMetricsSnapshot(mp.getConnection(), 1L); - HttpRequestMetricsSnapshot hRqSnap = new HttpRequestMetricsSnapshot(mp.getConnection(),1535065195000L); - parser.parseHttpMetrics("build/private/test_resources/dev/shm", 1535065195000L, 1535065200000L, hRqSnap); - parser.parseRequestMetrics("build/private/test_resources/dev/shm", 1535065195000L, 1535065200000L, rqSnap); - assertEquals(132, hRqSnap.fetchAll().size(), 0); - assertEquals(266, rqSnap.fetchAll().size(), 0); - } - - //@Test - public void perfTest() throws Exception { - ReaderMetricsProcessor mp = new ReaderMetricsProcessor("test_files/dev/shm"); - MetricsParser parser = new MetricsParser(); - OSMetricsSnapshot osMetricsSnap = new OSMetricsSnapshot(mp.getConnection(), 1L); - mp.getConnection().setAutoCommit(false); - long mCurrT = System.currentTimeMillis(); - parser.parseOSMetrics("test_files/dev/shm", 1537233539000L, 1537232364000L, osMetricsSnap); - long mFinalT = System.currentTimeMillis(); - System.out.println(mFinalT - mCurrT); - } - - @Test - public void testOSMetricRotateParse() throws Exception { - - try (CopyTestResource testResource = new CopyTestResource("build/private/test_resources/dev/shm", - "build/private/test_resources/dev/shm_metricsparser_testOSMetricRotateParse")) { - - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(testResource.getPath()); - MetricsParser parser = new MetricsParser(); - - OSMetricsSnapshot osMetricsSnap = new OSMetricsSnapshot(mp.getConnection(), 1L); - parser.parseOSMetrics(testResource.getPath(), 1535065195000L, 1535065200000L, osMetricsSnap); - assertEquals(136, osMetricsSnap.fetchAll().size(), 0); - - File file1 = new File(testResource.getPath() + "/1535065170000/threads/7611/os_metrics"); - File file2 = new File(testResource.getPath() + "/1535065170000/threads/6183/os_metrics"); - long orgModifiedTime1 = file1.lastModified(); - long orgModifiedTime2 = file2.lastModified(); - // set modified to higher than end time - file1.setLastModified(1535065200000L + 2000L); - // set modified to lower than start time - file2.setLastModified(1535065195000L - 2000L); - try { - osMetricsSnap = new OSMetricsSnapshot(mp.getConnection(), 2L); - parser.parseOSMetrics(testResource.getPath(), 1535065195000L, 1535065200000L, osMetricsSnap); - assertEquals(135, osMetricsSnap.fetchAll().size(), 0); - } catch (Exception e) { - assertTrue("unexpected exception" + e.getMessage(), false); - } finally { - file1.setLastModified(orgModifiedTime1); - file2.setLastModified(orgModifiedTime2); - } - } - } -} - - - diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/NewFormatProcessorTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/NewFormatProcessorTest.java new file mode 100644 index 00000000..ab9e7ddd --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/NewFormatProcessorTest.java @@ -0,0 +1,439 @@ +package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.EventLogFileHandler; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.EventLog; + +@Ignore +public class NewFormatProcessorTest { + private static final String DUMP_DIR = "/tmp"; + + @BeforeClass + public static void unzipFiles() { + // Unzip files in the /tmp directory. + String newFilesZipPath = "test_files/new_format/new_format.tar.gz"; + String oldFilesZipPath = "test_files/old_format/old_format.tar.gz"; + + String[] newFilesCreationCmd = {"tar", "-xf", newFilesZipPath, "-C", DUMP_DIR}; + String[] oldFilesCreationCmd = {"tar", "-xf", oldFilesZipPath, "-C", DUMP_DIR}; + + + if (!Files.exists(Paths.get(DUMP_DIR + "/old_format"))) { + try { + Runtime.getRuntime().exec(oldFilesCreationCmd); + } catch (IOException e) { + e.printStackTrace(); + } + } + + if (!Files.exists(Paths.get(DUMP_DIR + "/new_format"))) { + try { + Runtime.getRuntime().exec(newFilesCreationCmd); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + private static void removeFilesAndDir(Path path) throws IOException { + Files.walk(path) + .map(Path::toFile) + .sorted(Comparator.reverseOrder()) + .forEach(File::delete); + + } + // @AfterClass + public static void removeFiles() throws IOException { + removeFilesAndDir(Paths.get(DUMP_DIR + "/old_format")); + removeFilesAndDir(Paths.get(DUMP_DIR + "/new_format")); + } + + @Before + public void init() { + ESResources.INSTANCE.setPluginFileLocation(""); + } + + //@Test + public void processNewMetricsFormat() throws Exception { + Path path = Paths.get(System.getProperty("user.dir")); + Path configPath = Paths.get(path.toString(), "test_files/new_format/performance-analyzer.properties"); + System.out.println("==" + configPath + "=="); + System.setProperty("configFilePath", configPath.toString()); + + ReaderMetricsProcessor readerMetricsProcessor = new ReaderMetricsProcessor( + DUMP_DIR + "/new_format", true); + + EventLogFileHandler eventLogFileHandler = new EventLogFileHandler(new EventLog(), DUMP_DIR + "/new_format"); + // Map> metricsDataMap = + // eventLogFileHandler.read(String.valueOf(1566413970000L)); + readerMetricsProcessor.processMetrics(DUMP_DIR + "/new_format", 1566413970000L); + } + + //@Test + public void processOldMetricsFormat() throws Exception { + Path path = Paths.get(System.getProperty("user.dir")); + Path configPath = Paths.get(path.toString(), "test_files/old_format/performance-analyzer.properties"); + System.out.println("==" + configPath + "=="); + System.setProperty("configFilePath", configPath.toString()); + + ReaderMetricsProcessor readerMetricsProcessor = new ReaderMetricsProcessor(DUMP_DIR + "/old_format"); + readerMetricsProcessor.processMetrics(DUMP_DIR + "/old_format", 1566413970000L); + } + + private ReaderMetricsProcessorDummy createReaderMetricsProcessor(DATA_FORMAT format) throws Exception { + Path path = Paths.get(System.getProperty("user.dir")); + Path configPath = Paths.get(path.toString(), "test_files/" + format.format + "/performance-analyzer.properties"); + System.out.println("==" + configPath + "=="); + System.setProperty("configFilePath", configPath.toString()); + + return new ReaderMetricsProcessorDummy(format); + } + + // The idea is to call processNodeMetrics with the old format and the new format and compare the results. + @Test + public void nodeMetricsComparison() throws Exception { + ReaderMetricsProcessorDummy oldFormatProcessor = createReaderMetricsProcessor(DATA_FORMAT.OLD); + ReaderMetricsProcessorDummy newFormatProcessor = createReaderMetricsProcessor(DATA_FORMAT.NEW); + + newFormatProcessor.parseNodeMetrics(1566413970000L); + oldFormatProcessor.parseNodeMetrics(1566413970000L); + Map> oldFormatNodeMetricsMap = + oldFormatProcessor.getNodeMetricsMap(); + Map> newFormatNodeMetricsMap = + newFormatProcessor.getNodeMetricsMap(); + + compareNodeMetricsMaps(oldFormatNodeMetricsMap, newFormatNodeMetricsMap); + } + + //@Test + public void compareRequestMetricsSnapshots() throws Exception { + ReaderMetricsProcessorDummy oldFormatProcessor = createReaderMetricsProcessor(DATA_FORMAT.OLD); + ReaderMetricsProcessorDummy newFormatProcessor = createReaderMetricsProcessor(DATA_FORMAT.NEW); + + newFormatProcessor.parseRequestMetrics(1566413970000L); + oldFormatProcessor.parseRequestMetrics(1566413970000L); + NavigableMap oldFormatReqMetricsMap = + oldFormatProcessor.getShardRequestMetricsMap(); + NavigableMap newFormatReqMetricsMap = + newFormatProcessor.getShardRequestMetricsMap(); + + compareReqMetricsMaps(oldFormatReqMetricsMap, newFormatReqMetricsMap); + } + + //@Test + public void compareHttpRequestMetricsSnapshots() throws Exception { + ReaderMetricsProcessorDummy oldFormatProcessor = createReaderMetricsProcessor(DATA_FORMAT.OLD); + ReaderMetricsProcessorDummy newFormatProcessor = createReaderMetricsProcessor(DATA_FORMAT.NEW); + + oldFormatProcessor.parseHttpRequestMetrics(1566413970000L); + newFormatProcessor.parseHttpRequestMetrics(1566413970000L); + NavigableMap oldFormatReqMetricsMap = + oldFormatProcessor.getHttpRqMetricsMap(); + NavigableMap newFormatReqMetricsMap = + newFormatProcessor.getHttpRqMetricsMap(); + + compareHttpReqMetricsMaps(oldFormatReqMetricsMap, newFormatReqMetricsMap); + } + + //@Test + public void compareMasterRequestMetricsSnapshots() throws Exception { + ReaderMetricsProcessorDummy oldFormatProcessor = createReaderMetricsProcessor(DATA_FORMAT.OLD); + ReaderMetricsProcessorDummy newFormatProcessor = createReaderMetricsProcessor(DATA_FORMAT.NEW); + + oldFormatProcessor.parseMasterEventMetrics(1566413970000L); + newFormatProcessor.parseMasterEventMetrics(1566413970000L); + NavigableMap oldFormatReqMetricsMap = + oldFormatProcessor.getMasterEventMetricsMap(); + NavigableMap newFormatReqMetricsMap = + newFormatProcessor.getMasterEventMetricsMap(); + + compareMasterReqMetricsMaps(oldFormatReqMetricsMap, newFormatReqMetricsMap); + } + + //@Test + public void compareOsMetricsSnapshot() throws Exception { + ReaderMetricsProcessorDummy oldFormatProcessor = createReaderMetricsProcessor(DATA_FORMAT.OLD); + ReaderMetricsProcessorDummy newFormatProcessor = createReaderMetricsProcessor(DATA_FORMAT.NEW); + + newFormatProcessor.parseOsMetrics(1566413970000L); + oldFormatProcessor.parseOsMetrics(1566413970000L); + NavigableMap oldFormatNodeMetricsMap = + oldFormatProcessor.getOsMetricsMap(); + NavigableMap newFormatNodeMetricsMap = + newFormatProcessor.getOsMetricsMap(); + + compareOsMetricsMaps(oldFormatNodeMetricsMap, newFormatNodeMetricsMap); + } + + private void compareOsMetricsMaps(NavigableMap old, + NavigableMap nw) { + Assert.assertEquals(old.size(), nw.size()); + String oldFilename = "old.txt"; + String newFilename = "new.txt"; + for (NavigableMap.Entry oldEntry: old.entrySet()) { + OSMetricsSnapshot newSnap = nw.get(oldEntry.getKey()); + try { + PrintWriter oldf = new PrintWriter(oldFilename); + oldf.println(oldEntry.getValue().fetchAll()); + System.out.println("OLDD"); + System.out.println(oldEntry.getValue().fetchAll()); + + PrintWriter newf = new PrintWriter(newFilename); + newf.println(newSnap.fetchAll()); + System.out.println("NEWW"); + System.out.println(newSnap.fetchAll()); + } catch (FileNotFoundException e) { + e.printStackTrace(); + Assert.fail(); + } + compareFiles(oldFilename, newFilename); + } + + } + private void compareReqMetricsMaps(NavigableMap old, + NavigableMap nw) { + Assert.assertEquals(old.size(), nw.size()); + String oldFilename = "old.txt"; + String newFilename = "new.txt"; + for (NavigableMap.Entry oldEntry: old.entrySet()) { + ShardRequestMetricsSnapshot newSnap = nw.get(oldEntry.getKey()); + try { + PrintWriter oldf = new PrintWriter(oldFilename); + oldf.println(oldEntry.getValue().fetchAll()); + + PrintWriter newf = new PrintWriter(newFilename); + newf.println(newSnap.fetchAll()); + } catch (FileNotFoundException e) { + e.printStackTrace(); + Assert.fail(); + } + compareFiles(oldFilename, newFilename); + } + + } + + private void compareHttpReqMetricsMaps(NavigableMap old, + NavigableMap nw) { + Assert.assertEquals(old.size(), nw.size()); + String oldFilename = "old.txt"; + String newFilename = "new.txt"; + for (NavigableMap.Entry oldEntry: old.entrySet()) { + HttpRequestMetricsSnapshot newSnap = nw.get(oldEntry.getKey()); + try { + PrintWriter oldf = new PrintWriter(oldFilename); + oldf.println(oldEntry.getValue().fetchAll()); + + PrintWriter newf = new PrintWriter(newFilename); + newf.println(newSnap.fetchAll()); + } catch (FileNotFoundException e) { + e.printStackTrace(); + Assert.fail(); + } + compareFiles(oldFilename, newFilename); + } + + } + + private void compareMasterReqMetricsMaps(NavigableMap old, + NavigableMap nw) { + Assert.assertEquals(old.size(), nw.size()); + String oldFilename = "old.txt"; + String newFilename = "new.txt"; + for (NavigableMap.Entry oldEntry: old.entrySet()) { + MasterEventMetricsSnapshot newSnap = nw.get(oldEntry.getKey()); + try { + PrintWriter oldf = new PrintWriter(oldFilename); + oldf.println(oldEntry.getValue().fetchAll()); + + PrintWriter newf = new PrintWriter(newFilename); + newf.println(newSnap.fetchAll()); + } catch (FileNotFoundException e) { + e.printStackTrace(); + Assert.fail(); + } + compareFiles(oldFilename, newFilename); + } + + } + + private void compareFiles(String file1, String file2) { + Path p1 = Paths.get(file1); + Path p2 = Paths.get(file2); + + try { + List lines1 = Files.readAllLines(p1); + List lines2 = Files.readAllLines(p2); + Assert.assertEquals(lines1.size(), lines2.size()); + + for (int i=0; i < lines1.size(); i++) { + Assert.assertEquals(String.format("(%s)\n!=\n(%s)", lines1.get(i), lines2.get(i)), + lines1.get(i), lines2.get(i)); + } + Files.deleteIfExists(p1); + Files.deleteIfExists(p2); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private void compareNodeMetricsMaps(Map> old, + Map> nw) { + Assert.assertEquals(old.size(), nw.size()); + + for (Map.Entry> entryOld : old.entrySet()) { + System.out.println("=== " + entryOld.getKey().name() + " ===="); + for (Map.Entry snapOld: entryOld.getValue().entrySet()) { + Long kOld = snapOld.getKey(); + System.out.println("For time interval " + kOld); + MemoryDBSnapshot vOld = snapOld.getValue(); + MemoryDBSnapshot vNew = nw.get(entryOld.getKey()).get(kOld); + // Assert.assertEquals(v1, v2); + System.out.println("All OLD data"); + System.out.println(vOld.fetchAll()); + System.out.println("All NEW data"); + System.out.println(vNew.fetchAll()); + } + } + } + + + private enum DATA_FORMAT { + OLD("old_format"), + NEW("new_format"); + + String format; + + DATA_FORMAT(String format) { + this.format = format; + } + } + + static class ReaderMetricsProcessorDummy extends ReaderMetricsProcessor { + private final DATA_FORMAT format; + private final String rootLocation; + + ReaderMetricsProcessorDummy(DATA_FORMAT format) throws Exception { + super(DUMP_DIR + "/" + format.format, + format == DATA_FORMAT.NEW); + this.rootLocation = DUMP_DIR + "/" + format.format; + this.format = format; + } + + void processMetrics(long timestamp) throws Exception { + super.processMetrics(rootLocation, timestamp); + } + + public NodeMetricsEventProcessor parseNodeMetrics(long timestamp) throws Exception { + switch (format) { + case OLD: + // super.parseNodeMetrics(timestamp); + break; + case NEW: + long prevWindowEndTime = timestamp - MetricsConfiguration.SAMPLING_INTERVAL; + EventLogFileHandler eventLogFileHandler = new EventLogFileHandler(new EventLog(), rootLocation); + // Map> cmetricsDataMap = + // eventLogFileHandler.read(String.valueOf(timestamp)); + // Map> pmetricsDataMap = + // eventLogFileHandler.read(String.valueOf(prevWindowEndTime)); + // super.parseNodeMetrics(timestamp, cmetricsDataMap); + break; + } + return null; + } + + void parseOsMetrics(long timestamp) throws Exception { + long currWindowEndTime = PerformanceAnalyzerMetrics.getTimeInterval( + timestamp, MetricsConfiguration.SAMPLING_INTERVAL); + long prevWindowEndTime = currWindowEndTime - MetricsConfiguration.SAMPLING_INTERVAL; + switch (format) { + case OLD: + super.parseOSMetrics(rootLocation, currWindowEndTime, + currWindowEndTime+ MetricsConfiguration.SAMPLING_INTERVAL); + break; + case NEW: + // Map> currMetricsDataMap = + // getEventLogFileHandler().read(String.valueOf(currWindowEndTime)); + // Map> lastMetricsDataMap = + // getEventLogFileHandler().read(String.valueOf(prevWindowEndTime)); + + // super.parseOSMetrics(currWindowEndTime, + // currWindowEndTime + MetricsConfiguration.SAMPLING_INTERVAL, + // currMetricsDataMap); + break; + } + } + + void parseRequestMetrics(long timestamp) throws Exception { + long currWindowEndTime = PerformanceAnalyzerMetrics.getTimeInterval( + timestamp, MetricsConfiguration.SAMPLING_INTERVAL); + long prevWindowEndTime = currWindowEndTime - MetricsConfiguration.SAMPLING_INTERVAL; + switch (format) { + case OLD: + super.parseRequestMetrics(rootLocation, prevWindowEndTime, currWindowEndTime); + break; + case NEW: + // Map> currMetricsDataMap = + // getEventLogFileHandler().read(String.valueOf(prevWindowEndTime)); + // super.parseRequestMetrics(rootLocation, + // prevWindowEndTime, currWindowEndTim/ e, + // currMetricsDataMap); + break; + } + } + void parseHttpRequestMetrics(long timestamp) throws Exception { + long currWindowEndTime = PerformanceAnalyzerMetrics.getTimeInterval( + timestamp, MetricsConfiguration.SAMPLING_INTERVAL); + long prevWindowEndTime = currWindowEndTime - MetricsConfiguration.SAMPLING_INTERVAL; + switch (format) { + case OLD: + super.parseHttpRequestMetrics(rootLocation, prevWindowEndTime, currWindowEndTime); + break; + case NEW: + // Map> currMetricsDataMap = + // getEventLogFileHandler().read(String.valueOf(prevWindowEndTime)); + // super.parseHttpRequestMetrics(rootLocation, + // prevWindowEndTime, currWindowEndTime, + // currMetricsDataMap); + break; + } + } + MasterMetricsEventProcessor parseMasterEventMetrics(long timestamp) { + long currWindowEndTime = PerformanceAnalyzerMetrics.getTimeInterval( + timestamp, MetricsConfiguration.SAMPLING_INTERVAL); + long prevWindowEndTime = currWindowEndTime - MetricsConfiguration.SAMPLING_INTERVAL; + switch (format) { + case OLD: + super.parseMasterEventMetrics(rootLocation, prevWindowEndTime, currWindowEndTime); + break; + case NEW: + // Map> currMetricsDataMap = + // getEventLogFileHandler().read(String.valueOf(prevWindowEndTime)); + // super.parseMasterEventMetrics(rootLocation, prevWindowEndTime, currWindowEndTime, + // currMetricsDataMap); + break; + } + return null; + } + } +} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/OSMetricsSnapshotTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/OSMetricsSnapshotTests.java deleted file mode 100644 index d964b5c9..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/OSMetricsSnapshotTests.java +++ /dev/null @@ -1,306 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import static org.junit.Assert.assertEquals; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.util.HashMap; -import java.util.Map; - -import org.jooq.BatchBindStep; -import org.jooq.Record; -import org.jooq.Result; -import org.junit.Test; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.OSMetrics; - -@SuppressWarnings("serial") -public class OSMetricsSnapshotTests { - private static final String DB_URL = "jdbc:sqlite:"; - - public OSMetricsSnapshotTests() throws ClassNotFoundException { - Class.forName("org.sqlite.JDBC"); - System.setProperty("java.io.tmpdir", "/tmp"); - } - - //@Test - public void perfTest() throws Exception { - System.out.println("Batch Insert"); - System.out.println("100: "+runBatchTest(100, 1)); - System.out.println("1000: "+runBatchTest(1000, 1)); - System.out.println("10000: "+runBatchTest(10000, 1)); - System.out.println("300000: "+runBatchTest(300000, 1)); - Connection conn = DriverManager.getConnection(DB_URL); - conn.setAutoCommit(false); - OSMetricsSnapshot osMetricsSnap = new OSMetricsSnapshot(conn, 1L); - - Thread t1 = new Thread(new Runnable() { - @Override - public void run() { - try { - System.out.println("100000: "+runBatchTest(100000,osMetricsSnap)); - } catch(Exception e) { - } - } - }); - t1.start(); - - Thread t2 = new Thread(new Runnable() { - @Override - public void run() { - try { - System.out.println("100000: "+runBatchTest(100000,osMetricsSnap)); - } catch(Exception e) { - } - } - }); - - Thread t3 = new Thread(new Runnable() { - @Override - public void run() { - try { - System.out.println("100000: "+runBatchTest(100000,osMetricsSnap)); - } catch(Exception e) { - } - } - }); - - t2.start(); - t3.start(); - t1.join(); - t2.join(); - t3.join(); - conn.commit(); - } - - //@Test - public void perfTestDifferentConnections() throws Exception { - System.out.println("Batch Insert"); - System.out.println("100: "+runBatchTest(100, 1)); - System.out.println("1000: "+runBatchTest(1000, 1)); - System.out.println("10000: "+runBatchTest(10000, 1)); - //System.out.println("100000: "+runBatchTest(100000)); - Thread t1 = new Thread(new Runnable() { - @Override - public void run() { - try { - System.out.println("100000: "+runBatchTest(100000,1L)); - } catch(Exception e) { - } - } - }); - t1.start(); - - Thread t2 = new Thread(new Runnable() { - @Override - public void run() { - try { - System.out.println("100000: "+runBatchTest(100000,1L)); - } catch(Exception e) { - } - } - }); - - Thread t3 = new Thread(new Runnable() { - @Override - public void run() { - try { - System.out.println("100000: "+runBatchTest(100000,1L)); - } catch(Exception e) { - } - } - }); - - t2.start(); - t3.start(); - t1.join(); - t2.join(); - t3.join(); - } - - - private Long runBatchTest(int iterations, OSMetricsSnapshot osMetricsSnap) throws Exception { - Map dimensions = new HashMap<>(); - AllMetrics.OSMetrics[] metrics = AllMetrics.OSMetrics.values(); - int numMetrics = metrics.length + 2; - Object [] metricVals = new Object[numMetrics]; - metricVals[0] = "1"; - metricVals[1] = "GC"; - - Map metricsMap = new HashMap() {{ - this.put("avgReadSyscallRate", 100d); - this.put("cpu", 13223.323243d); - this.put("runtime", 22222d); - this.put("heap_usage",444d); - this.put("waittime", 2132134d); - this.put("ctxrate", 3243.21321d); - this.put("avgTotalSyscallRate", 32432.324d); - this.put("rss", 23432d); - this.put("paging_majflt", 32432432d); - this.put("avgWriteThroughputBps", 32423d); - this.put("avgWriteSyscallRate", 234324.3432d); - this.put("avgTotalThroughputBps", 324323432d); - this.put("avgReadThroughputBps", 2342343223d); - this.put("paging_minflt", 23432.32432d); - }}; - for(int i=2;i dimensions = new HashMap<>(); - AllMetrics.OSMetrics[] metrics = AllMetrics.OSMetrics.values(); - int numMetrics = metrics.length + 2; - Object [] metricVals = new Object[numMetrics]; - metricVals[0] = "1"; - metricVals[1] = "GC"; - - Map metricsMap = new HashMap() {{ - this.put("avgReadSyscallRate", 100d); - this.put("cpu", 13223.323243d); - this.put("runtime", 22222d); - this.put("heap_usage",444d); - this.put("waittime", 2132134d); - this.put("ctxrate", 3243.21321d); - this.put("avgTotalSyscallRate", 32432.324d); - this.put("rss", 23432d); - this.put("paging_majflt", 32432432d); - this.put("avgWriteThroughputBps", 32423d); - this.put("avgWriteSyscallRate", 234324.3432d); - this.put("avgTotalThroughputBps", 324323432d); - this.put("avgReadThroughputBps", 2342343223d); - this.put("paging_minflt", 23432.32432d); - }}; - for(int i=2;i metrics = new HashMap<>(); - Map dimensions = new HashMap<>(); - dimensions.put("tid", "1"); - dimensions.put("tName", "dummy thread"); - metrics.put(OSMetrics.CPU_UTILIZATION.toString(), 2.3333d); - metrics.put(OSMetrics.PAGING_RSS.toString(), 3.63d); - osMetricsSnap.putMetric(metrics, dimensions, 7000L); - dimensions.put("tid", "2"); - metrics.put(OSMetrics.CPU_UTILIZATION.toString(), 5.0d); - metrics.put(OSMetrics.PAGING_RSS.toString(), 3.63d); - osMetricsSnap.putMetric(metrics, dimensions, 7000L); - OSMetricsSnapshot os2 = new OSMetricsSnapshot(conn, 12000L); - dimensions.put("tid", "1"); - metrics.put(OSMetrics.CPU_UTILIZATION.toString(), 2.3333d); - metrics.put(OSMetrics.PAGING_RSS.toString(), 3.63d); - os2.putMetric(metrics, dimensions, 12000L); - dimensions.put("tid", "2"); - metrics.put(OSMetrics.CPU_UTILIZATION.toString(), 3.0d); - metrics.put(OSMetrics.PAGING_RSS.toString(), 3.63d); - os2.putMetric(metrics, dimensions, 12000L); - dimensions.put("tid", "3"); - metrics.put(OSMetrics.CPU_UTILIZATION.toString(), 3.0d); - metrics.put(OSMetrics.PAGING_RSS.toString(), 3.63d); - os2.putMetric(metrics, dimensions, 12000L); - - - OSMetricsSnapshot osFinal = new OSMetricsSnapshot(conn, 3L); - OSMetricsSnapshot.alignWindow(osMetricsSnap, os2, - osFinal.getTableName(),5000L, 10000L); - Result res = osFinal.fetchAll(); - //System.out.println(res); - Double cpu = Double.parseDouble(res.get(0).get(OSMetrics.CPU_UTILIZATION.toString()).toString()); - assertEquals(cpu.doubleValue(), 2.3333d, 0); - cpu = Double.parseDouble(res.get(1).get(OSMetrics.CPU_UTILIZATION.toString()).toString()); - assertEquals(cpu.doubleValue(), 3.8d, 0); - } - - @Test - public void testAlignWindow() throws Exception { - // - Connection conn = DriverManager.getConnection(DB_URL); - OSMetricsSnapshot osMetricsSnap = new OSMetricsSnapshot(conn, 5000L); - //Create OSMetricsSnapshot - Map metrics = new HashMap<>(); - Map dimensions = new HashMap<>(); - dimensions.put("tid", "1"); - dimensions.put("tName", "dummy thread"); - metrics.put("CPU_Utilization", 10d); - osMetricsSnap.putMetric(metrics, dimensions, 7000L); - dimensions.put("tid", "2"); - metrics.put("CPU_Utilization", 20d); - osMetricsSnap.putMetric(metrics, dimensions, 8000L); - OSMetricsSnapshot os2 = new OSMetricsSnapshot(conn, 10000L); - dimensions.put("tid", "1"); - metrics.put("CPU_Utilization", 20d); - os2.putMetric(metrics, dimensions, 13000L); - dimensions.put("tid", "3"); - metrics.put("CPU_Utilization", 30d); - os2.putMetric(metrics, dimensions, 12000L); - - OSMetricsSnapshot osFinal = new OSMetricsSnapshot(conn, 3L); - OSMetricsSnapshot.alignWindow(osMetricsSnap, os2, - osFinal.getTableName(),5000L, 10000L); - Result res = osFinal.fetchAll(); - assertEquals(3, res.size()); - //System.out.println(res); - Double cpu = Double.parseDouble(res.get(0).get("CPU_Utilization").toString()); - assertEquals(cpu.doubleValue(), 16d, 0); - cpu = Double.parseDouble(res.get(1).get("CPU_Utilization").toString()); - assertEquals(cpu.doubleValue(), 20, 0); - cpu = Double.parseDouble(res.get(2).get("CPU_Utilization").toString()); - assertEquals(cpu.doubleValue(), 30, 0); - } -} - - diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ParseNodeMetricsTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ParseNodeMetricsTests.java deleted file mode 100644 index c025442c..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ParseNodeMetricsTests.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.sql.SQLException; -import java.util.Map; -import java.util.NavigableMap; - -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.impl.DSL; -import org.junit.Test; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.DiskDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.DiskValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricName; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; - -public class ParseNodeMetricsTests extends AbstractReaderTests { - long currentTimeMills = System.currentTimeMillis(); - public ParseNodeMetricsTests() throws SQLException, ClassNotFoundException { - super(); - } - - @Test - public void testParseMasterPendingMetrics() throws Exception { - long currTimestamp = System.currentTimeMillis() + 94000; - - long currTimeBucket = PerformanceAnalyzerMetrics.getTimeInterval(currTimestamp); - String currentTimeBucketStr = String.valueOf(currTimeBucket); - temporaryFolder.newFolder(currentTimeBucketStr, - PerformanceAnalyzerMetrics.sPendingTasksPath, PerformanceAnalyzerMetrics.MASTER_CURRENT); - File output = temporaryFolder.newFile(createRelativePath( - currentTimeBucketStr, PerformanceAnalyzerMetrics.sPendingTasksPath, - PerformanceAnalyzerMetrics.MASTER_CURRENT, PerformanceAnalyzerMetrics.MASTER_META_DATA)); - long lastUpdatedTime = System.currentTimeMillis(); - - int pendingTasksCount = 0; - write(output, false, getCurrentMilliSeconds(lastUpdatedTime), - createPendingTaskMetrics(pendingTasksCount)); - - MetricProperties masterPendingProperty = MetricPropertiesConfig - .getInstance().getProperty(MetricName.MASTER_PENDING); - - - masterPendingProperty.getHandler().setRootLocation(rootLocation); - - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - mp.parseNodeMetrics(currTimestamp); - - for (Map.Entry> entry : mp - .getNodeMetricsMap().entrySet()) { - NavigableMap curMap = entry.getValue(); - if (entry.getKey() != MetricName.MASTER_PENDING) { - assertTrue(curMap.isEmpty()); - continue; - } - - assertTrue(1 == curMap.size()); - Map.Entry firstEntry = curMap.lastEntry(); - assertTrue(lastUpdatedTime == firstEntry.getKey()); - - MemoryDBSnapshot curSnap = firstEntry.getValue(); - Result res = curSnap.fetchAll(); - assertTrue(1 == res.size()); - - Field field = DSL.field( - MasterPendingValue.MASTER_PENDING_QUEUE_SIZE.toString(), - Double.class); - - Record record0 = res.get(0); - - Double pendingCount = Double - .parseDouble(record0.get(field).toString()); - assertEquals(0, pendingCount, 0.001); - - } - - mp.trimOldSnapshots(); - mp.deleteDBs(); - } - - @Test - public void testParseDiskMetrics() throws Exception { - long currTimestamp = System.currentTimeMillis() + 94000; - - long currTimeBucket = PerformanceAnalyzerMetrics.getTimeInterval(currTimestamp); - String currentTimeBucketStr = String.valueOf(currTimeBucket); - temporaryFolder.newFolder(currentTimeBucketStr); - File output = temporaryFolder.newFile(createRelativePath( - currentTimeBucketStr, PerformanceAnalyzerMetrics.sDisksPath)); - long lastUpdatedTime = System.currentTimeMillis(); - - String diskXvda = "xvda"; - String diskNvme0n1 = "nvme0n1"; - double util1 = 0.0008d; - double wait1 = 2.0d; - double srate1 = 14.336d; - double util2 = 0.0d; - double wait2 = 0.0d; - double srate2 = 0.0d; - write(output, false, getCurrentMilliSeconds(lastUpdatedTime), - createDiskMetrics(diskXvda, util1, wait1, srate1), - createDiskMetrics(diskNvme0n1, 0.0d, 0.0d, 0.0)); - - MetricProperties diskProperty = MetricPropertiesConfig.getInstance() - .getProperty(MetricName.DISK_METRICS); - - String rootLocation = temporaryFolder.getRoot().getCanonicalPath() - + File.separator; - diskProperty.getHandler().setRootLocation(rootLocation); - - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - mp.parseNodeMetrics(currTimestamp); - - verifyDiskMetrics(mp.getNodeMetricsMap(), lastUpdatedTime, diskXvda, - diskNvme0n1, util1, wait1, srate1, util2, wait2, srate2, 1); - - util1 = 0.0009d; - wait1 = 2.1d; - srate1 = 14.436d; - util2 = 0.1009d; - wait2 = 0.0d; - srate2 = 0.0d; - - lastUpdatedTime = System.currentTimeMillis(); - - write(output, false, getCurrentMilliSeconds(lastUpdatedTime), - createDiskMetrics(diskXvda, util1, wait1, srate1), - createDiskMetrics(diskNvme0n1, util2, wait2, srate2)); - - currTimestamp = System.currentTimeMillis() + 94000; - mp.parseNodeMetrics(currTimestamp); - - verifyDiskMetrics(mp.getNodeMetricsMap(), lastUpdatedTime, diskXvda, - diskNvme0n1, util1, wait1, srate1, util2, wait2, srate2, 2); - - mp.trimOldSnapshots(); - mp.deleteDBs(); - } - - private void verifyDiskMetrics( - Map> nodeMetricsMap, - long lastUpdatedTime, String diskXvda, String diskNvme0n1, - double util1, double wait1, double srate1, double util2, - double wait2, double srate2, int expectedDiskMapSize) { - for (Map.Entry> entry : nodeMetricsMap - .entrySet()) { - NavigableMap curMap = entry.getValue(); - if (entry.getKey() != MetricName.DISK_METRICS) { - assertTrue(curMap.isEmpty()); - continue; - } - - assertTrue(expectedDiskMapSize == curMap.size()); - Map.Entry firstEntry = curMap.lastEntry(); - assertTrue(lastUpdatedTime == firstEntry.getKey()); - - MemoryDBSnapshot curSnap = firstEntry.getValue(); - assertTrue(2 == curSnap.fetchAll().size()); - - @SuppressWarnings("unchecked") - Field[] fields = new Field[3]; - - fields[0] = DSL.field(DiskValue.DISK_UTILIZATION.toString(), Double.class); - fields[1] = DSL.field(DiskValue.DISK_WAITTIME.toString(), Double.class); - fields[2] = DSL.field(DiskValue.DISK_SERVICE_RATE.toString(), Double.class); - - Result resRecordDiskXvda = curSnap.fetchMetric( - getDimensionEqCondition(DiskDimension.DISK_NAME, diskXvda), - fields); - - assertTrue(1 == resRecordDiskXvda.size()); - - Record record0 = resRecordDiskXvda.get(0); - - Double util = Double.parseDouble(record0.get(fields[0]).toString()); - assertEquals(util1, util, 0.001); - - Double wait = Double.parseDouble(record0.get(fields[1]).toString()); - assertEquals(wait1, wait, 0.001); - - Double srate = Double - .parseDouble(record0.get(fields[2]).toString()); - assertEquals(srate1, srate, 0.001); - - Result resRecordNvme0n1 = curSnap.fetchMetric( - getDimensionEqCondition(DiskDimension.DISK_NAME, diskNvme0n1), - fields); - - assertTrue(1 == resRecordDiskXvda.size()); - - Record record1 = resRecordNvme0n1.get(0); - - util = Double.parseDouble(record1.get(fields[0]).toString()); - assertEquals(util2, util, 0.001); - - wait = Double.parseDouble(record1.get(fields[1]).toString()); - assertEquals(wait2, wait, 0.001); - - srate = Double.parseDouble(record1.get(fields[2]).toString()); - assertEquals(srate2, srate, 0.001); - } - } - - -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessorTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessorTest.java deleted file mode 100644 index ffe0d7d2..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessorTest.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.util.CopyTestResource; -import org.jooq.Record; -import org.jooq.Result; -import org.junit.Test; - -import java.io.File; -import java.io.FilenameFilter; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -import static org.junit.Assert.assertEquals; - -// This test will be skipped in gradle -public class ReaderMetricsProcessorTest extends AbstractReaderTests { - - public ReaderMetricsProcessorTest() throws SQLException, ClassNotFoundException { - super(); - } - - @Test - public void testReaderMetricsProcessorFrequently() throws Exception { - - try (CopyTestResource testResource = new CopyTestResource("build/private/test_resources/dev/shm", - "build/private/test_resources/dev/shm_processor_testFrequently")) { - String rootLocation = testResource.getPath(); - deleteAll(); - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - - for (long i = 1535065139000L; i <= 1535065326500L; i += 2500) { - Map orgModified = new HashMap<>(); - final long time = i; - if (i % 1000 == 0) { - walkFiles(rootLocation, time, (File file) -> { - orgModified.put(file.getAbsolutePath(), file.lastModified()); - return file.setLastModified(time - 2000); - }); - } - - mp.processMetrics(rootLocation, i); - - if (i % 1000 == 0) { - walkFiles(rootLocation, time, (File file) -> - file.setLastModified(orgModified.get(file.getAbsolutePath()))); - } - } - - Result res = mp.getMetricsDB().getValue().queryMetric(Arrays.asList("CPU_Utilization"), - Arrays.asList("sum"), - Arrays.asList("ShardID", "IndexName", "Operation")); - Double shardFetchCpu = 0d; - for (Record record : res) { - if (record.get("Operation").equals("shardfetch")) { - shardFetchCpu = Double.parseDouble(record.get("CPU_Utilization").toString()); - break; - } - } - - assertEquals(0D, shardFetchCpu.doubleValue(), 0.001); - - mp.trimOldSnapshots(); - mp.deleteDBs(); - } - } - - @Test - public void testReaderMetricsProcessorFrequentlyWithDelay() throws Exception { - try (CopyTestResource testResource = new CopyTestResource("build/private/test_resources/dev/shm", - "build/private/test_resources/dev/shm_processor_testFrequentlyWithDelay")) { - - String rootLocation = testResource.getPath(); - deleteAll(); - int delay = 0; - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - - for (long i = 1535065139000L; i <= 1535065326500L; i += 2500) { - Map orgModified = new HashMap<>(); - final long time = i; - if (i % 1000 == 0) { - walkFiles(rootLocation, time, (File file) -> { - orgModified.put(file.getAbsolutePath(), file.lastModified()); - return file.setLastModified(time - 2000); - }); - } - - mp.processMetrics(rootLocation, i + delay); - delay = (delay + 1000) % 4000; - - if (i % 1000 == 0) { - walkFiles(rootLocation, time, (File file) -> - file.setLastModified(orgModified.get(file.getAbsolutePath()))); - } - } - - Result res = mp.getMetricsDB().getValue().queryMetric(Arrays.asList("CPU_Utilization"), - Arrays.asList("sum"), - Arrays.asList("ShardID", "IndexName", "Operation")); - Double shardFetchCpu = 0d; - for (Record record : res) { - if (record.get("Operation").equals("shardfetch")) { - shardFetchCpu = Double.parseDouble(record.get("CPU_Utilization").toString()); - break; - } - } - - assertEquals(0D, shardFetchCpu.doubleValue(), 0.001); - - mp.trimOldSnapshots(); - mp.deleteDBs(); - } - } - - - public void deleteAll() { - final File folder = new File("/tmp"); - final File[] files = folder.listFiles(new FilenameFilter() { - @Override - public boolean accept(final File dir, final String name) { - return name.matches("metricsdb_.*"); - } - }); - for (final File file : files) { - if (!file.delete()) { - System.err.println("Can't remove " + file.getAbsolutePath()); - } - } - } - - private void walkFiles(String rootLocation, long time, Function fun) { - long startTimeThirtySecondBucket = time / 30000 * 30000; - - File threadsFile = new File(rootLocation + File.separator - + startTimeThirtySecondBucket + File.separator - + "threads"); - - if (threadsFile == null || threadsFile.listFiles() == null) { - return; - } - - for (File threadIDFile : threadsFile.listFiles()) { - if (!threadIDFile.getName().equals("http")) { - - for (File opFile : threadIDFile.listFiles()) { - if (opFile.getName().equals("os_metrics")) { - fun.apply(opFile); - } - } - } - } - } - -} - diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessorTests.java deleted file mode 100644 index cfe25934..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessorTests.java +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FilenameFilter; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.NavigableMap; -import java.util.TreeMap; - -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.impl.DSL; -import org.junit.Test; -import org.mockito.Mockito; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CommonDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingValue; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricName; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.OSMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; - -public class ReaderMetricsProcessorTests extends AbstractReaderTests { - - public ReaderMetricsProcessorTests() throws SQLException, ClassNotFoundException { - super(); - } - - // Disabled on purpose - // @Test - public void testReaderMetricsProcessor() throws Exception { - String rootLocation = "build/private/test_resources/dev/shm"; - deleteAll(); - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - mp.processMetrics(rootLocation, 1535065139000L); - mp.processMetrics(rootLocation, 1535065169000L); - mp.processMetrics(rootLocation, 1535065199000L); - mp.processMetrics(rootLocation, 1535065229000L); - mp.processMetrics(rootLocation, 1535065259000L); - mp.processMetrics(rootLocation, 1535065289000L); - mp.processMetrics(rootLocation, 1535065319000L); - mp.processMetrics(rootLocation, 1535065349000L); - Result res = mp.getMetricsDB().getValue().queryMetric(Arrays.asList(OSMetrics.CPU_UTILIZATION.toString()), - Arrays.asList("sum"), - Arrays.asList(CommonDimension.SHARD_ID.toString(), - CommonDimension.INDEX_NAME.toString(), - CommonDimension.OPERATION.toString())); - Double shardFetchCpu = 0d; - for (Record record: res) { - if (record.get(CommonDimension.OPERATION.toString()).equals("shardfetch")) { - shardFetchCpu = Double.parseDouble(record.get(OSMetrics.CPU_UTILIZATION.toString()).toString()); - break; - } - } - assertEquals(0.0016D, shardFetchCpu.doubleValue(), 0.0001); - - mp.trimOldSnapshots(); - mp.deleteDBs(); - } - - public void deleteAll() { - final File folder = new File("/tmp"); - final File[] files = folder.listFiles(new FilenameFilter() { - @Override - public boolean accept(final File dir, final String name) { - return name.matches("metricsdb_.*"); - } - }); - for (final File file : files) { - if (!file.delete()) { - System.err.println("Can't remove " + file.getAbsolutePath()); - } - } - } - - private NavigableMap setUpAligningWindow( - long lastUpdateTime3) - throws Exception { - // time line - // writer writes to the left window at 2000l - // reader reads at 6001l - // writer writes to the right window at 7000l - // reader reads at 11001l - // writer writes to the right window at 12000l - // reader reads at 16001l - MemoryDBSnapshot masterPendingSnap1 = new MemoryDBSnapshot(conn, - MetricName.MASTER_PENDING, 6001L); - long lastUpdateTime1 = 2000L; - masterPendingSnap1.setLastUpdatedTime(lastUpdateTime1); - Object[][] values1 = { { 0 } }; - masterPendingSnap1.insertMultiRows(values1); - - MemoryDBSnapshot masterPendingSnap2 = new MemoryDBSnapshot(conn, - MetricName.MASTER_PENDING, 11001L); - long lastUpdateTime2 = 7000L; - masterPendingSnap2.setLastUpdatedTime(lastUpdateTime2); - Object[][] values2 = { { 1 } }; - masterPendingSnap2.insertMultiRows(values2); - - MemoryDBSnapshot masterPendingSnap3 = new MemoryDBSnapshot(conn, - MetricName.MASTER_PENDING, 16001L ); - masterPendingSnap2.setLastUpdatedTime(lastUpdateTime3); - Object[][] values3 = { { 3 } }; - masterPendingSnap3.insertMultiRows(values3); - - NavigableMap metricMap = new TreeMap<>(); - metricMap.put(lastUpdateTime1, masterPendingSnap1); - metricMap.put(lastUpdateTime2, masterPendingSnap2); - metricMap.put(lastUpdateTime3, masterPendingSnap3); - - return metricMap; - } - - private NavigableMap setUpAligningWindow() - throws Exception { - return setUpAligningWindow(12000L); - } - - /** - * Time line - * + writer writes 0 to the left window at 2000l - * + reader reads at 6001l - * + writer writes 1 to the right window at 7000l - * + reader reads at 11001l - * + writer writes 3 to the right window at 12000l - * + reader reads at 16001l - * - * Given metrics in two writer windows calculates a new reader window which - * overlaps with the given windows. - * |------leftWindow-------|-------rightWindow--------| - * 7000 - * 5000 100000 - * |-----------alignedWindow———| - * - * We retrieve left and right window using a metric map, whose key is the - * largest last modification time. - * leftWindow = metricsMap.get(7000) = 1 - * rightWindow = metricsMap.get(12000) = 3 - * - * We use values in the future to represent values in the past. So if at - * t1, writer writes values 1, the interval [t1-sample interval, t1] has - * value 1. - * So [2000, 7000] maps to 1, and [7000, 12000] maps to 3. We end up - * having - * (1 * 2000 + 3 * 3000) / 5000 = 2.2 - * - * @throws Exception - */ - @Test - public void testAlignNodeMetrics() throws Exception { - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - long readerTime1 = 6001L; - long readerTime2 = 11001L; - NavigableMap metricMap = setUpAligningWindow( - ); - // The 3rd parameter is windowEndTime. - // So we compute algined metrics based previous reader window [6000L, - // 11000l]. But we use PerformanceAnalyzerMetrics.getTimeInterval to - // compute the aligned reader window time: 10000. - // So our aligned window time is [5000,10000]. - MemoryDBSnapshot masterPendingFinal = new MemoryDBSnapshot(conn, - MetricName.MASTER_PENDING, - PerformanceAnalyzerMetrics.getTimeInterval(readerTime2, - MetricsConfiguration.SAMPLING_INTERVAL), - true); - - MemoryDBSnapshot alignedWindow = mp.alignNodeMetrics( - MetricName.MASTER_PENDING, metricMap, - PerformanceAnalyzerMetrics.getTimeInterval(readerTime1, - MetricsConfiguration.SAMPLING_INTERVAL), - PerformanceAnalyzerMetrics.getTimeInterval(readerTime2, - MetricsConfiguration.SAMPLING_INTERVAL), - masterPendingFinal); - - Result res = alignedWindow.fetchAll(); - assertTrue(1 == res.size()); - Field valueField = DSL.field( - MasterPendingValue.MASTER_PENDING_QUEUE_SIZE.toString(), Double.class); - Double pending = Double - .parseDouble(res.get(0).get(valueField).toString()); - assertEquals(2.2d, pending, 0.001); - } - - @Test - public void testEmitNodeMetrics() throws Exception { - // the Connection that the test uses and ReaderMetricsProcessor uses are - // different. - // Need to use the same one otherwise table created in the test won't be - // visible in ReaderMetricsProcessor. - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - ReaderMetricsProcessor spyMp = Mockito.spy(mp); - Mockito.doReturn(this.conn).when(spyMp).getConnection(); - - long readerTime2 = 11001L; - NavigableMap metricMap = setUpAligningWindow( - ); - - spyMp.putNodeMetricsMap(MetricName.MASTER_PENDING, metricMap); - - MetricsDB db = new MetricsDB(1553713512); - spyMp.emitNodeMetrics( - PerformanceAnalyzerMetrics.getTimeInterval(readerTime2, - MetricsConfiguration.SAMPLING_INTERVAL), db); - - Result res = db - .queryMetric(MasterPendingValue.MASTER_PENDING_QUEUE_SIZE.toString()); - - assertTrue(1 == res.size()); - - Record row0 = res.get(0); - for (int i=0; i metricMap = setUpAligningWindow( - ); - - spyMp.putNodeMetricsMap(MetricName.MASTER_PENDING, metricMap); - - MetricsDB db = new MetricsDB(1553713518); - spyMp.emitNodeMetrics( - PerformanceAnalyzerMetrics.getTimeInterval(readerTime2, - MetricsConfiguration.SAMPLING_INTERVAL), db); - - Result res = db - .queryMetric(MasterPendingValue.MASTER_PENDING_QUEUE_SIZE.toString()); - - assertTrue(1 == res.size()); - - Record row0 = res.get(0); - for (int i=0; i metricMap = setUpAligningWindow( - 9999L); - - spyMp.putNodeMetricsMap(MetricName.MASTER_PENDING, metricMap); - - MetricsDB db = new MetricsDB(1553713524); - spyMp.emitNodeMetrics(PerformanceAnalyzerMetrics.getTimeInterval(readerTime2, - MetricsConfiguration.SAMPLING_INTERVAL), db); - - assertTrue(!db.metricExists( - MasterPendingValue.MASTER_PENDING_QUEUE_SIZE.toString())); - db.remove(); - } -} - diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderTrimDBTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderTrimDBTests.java deleted file mode 100644 index 297e4e3f..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderTrimDBTests.java +++ /dev/null @@ -1,156 +0,0 @@ -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Vector; -import java.util.concurrent.ConcurrentSkipListMap; - -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -/** - * Tests to make sure that the MetricsDB files are correctly deleted. - */ -public class ReaderTrimDBTests { - NavigableMap metricsDBMap; - private Vector fileNames; - - @Before - public void setUp() { - metricsDBMap = new ConcurrentSkipListMap<>(); - fileNames = new Vector<>(); - System.setProperty("java.io.tmpdir", "/tmp"); - } - - @After - public void tearDown() { - for (String file : fileNames) { - File dbFile = new File(file); - } - } - - private void createDB(long startTime) throws Exception { - MetricsDB db = new MetricsDB(startTime); - metricsDBMap.put(startTime, new MetricsDB(startTime)); - fileNames.add(db.getDBFilePath()); - - } - - private void createNDBs(int start, int number) throws Exception { - for (long i = start; i < start + number; i++) { - createDB(i); - } - } - - /** - * The trimDatabase removes all the in-memory entries from the map provided to it as an argument, leaving behind - * the maxFiles number of entries. This essentially tests that. Deleting the in-memory entries may or may not - * clean up the files from the disk. This is controlled by the the third argument. If set to false, the the files - * on disk are left behind. So essentially, no files will be deleted. - * - * @throws Exception if creating metricsDB file fails - */ - @Test - public void testMetricsDBFilesNotCleaned() throws Exception { - int maxFiles = 2; - int numDBs = 10; - - createNDBs(0, numDBs); - assertEquals(numDBs, metricsDBMap.size()); - assertEquals(fileNames.size(), numDBs); - - // Goal is to clean up the in-memory entries leaving behind the maxFiles most recent ones, but to leave the - // on disk files untouched. - ReaderMetricsProcessor.trimDatabases(metricsDBMap, maxFiles, false); - - // Because deleteDBFiles is set to false, we expect no files to be deleted. - for (String file : fileNames) { - File dbFile = new File(file); - assertTrue(dbFile.exists()); - } - - // We check to see that only maxFiles entries are left. - assertEquals(metricsDBMap.size(), maxFiles); - - // Add a few more files. - createNDBs(numDBs, numDBs); - assertEquals(numDBs + maxFiles, metricsDBMap.size()); - assertEquals(numDBs * 2, fileNames.size()); - - // All the old and the newly created on disk files should not have been cleaned up by this call. - ReaderMetricsProcessor.trimDatabases(metricsDBMap, maxFiles, false); - - // Because deleteDBFiles is set to false, we expect no files to be deleted. - for (String file : fileNames) { - File dbFile = new File(file); - assertTrue(dbFile.exists()); - } - - // We check to see that only maxFiles entries are left. - assertEquals(metricsDBMap.size(), maxFiles); - } - - /** - * A counterpart of the testMetricsDBFilesNotCleaned test but checks that the files are cleaned up when the - * deleteDBFiles boolean is set to true. - * - * @throws Exception if creating metricsDB file fails - */ - @Test - public void testMetricsDBFilesCleaned() throws Exception { - int maxFiles = 2; - int numDBs = 10; - - createNDBs(0, numDBs); - assertEquals(metricsDBMap.size(), numDBs); - assertEquals(fileNames.size(), numDBs); - - // The idea is to clean up the in memory entries and the on disk files leaving behind only the maxFiles - // number of entries. - ReaderMetricsProcessor.trimDatabases(metricsDBMap, maxFiles, true); - - // Because deleteDBFiles is set to true, we expect only maxFiles number of files remaining. - int count = 0; - for (String file : fileNames) { - File dbFile = new File(file); - if (count < numDBs - maxFiles) { - assertFalse(dbFile.exists()); - } else { - assertTrue(dbFile.exists()); - } - count++; - } - - assertEquals(metricsDBMap.size(), maxFiles); - fileNames.removeAllElements(); - - for (Map.Entry pair : metricsDBMap.entrySet()) { - fileNames.add(pair.getValue().getDBFilePath()); - } - - createNDBs(numDBs, numDBs); - assertEquals(metricsDBMap.size(), numDBs + maxFiles); - assertEquals(fileNames.size(), numDBs + maxFiles); - - ReaderMetricsProcessor.trimDatabases(metricsDBMap, maxFiles, true); - - count = 0; - // Because deleteDBFiles is set to true, we expect only maxFiles number of files remaining. - for (String file : fileNames) { - File dbFile = new File(file); - if (count < numDBs) { - assertFalse("NOT expected to find file: " + file, dbFile.exists()); - } else { - assertTrue("Expected to find file: " + file, dbFile.exists()); - } - count++; - } - } -} diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ShardRequestMetricsSnapshotTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ShardRequestMetricsSnapshotTests.java deleted file mode 100644 index 67f7b6fc..00000000 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ShardRequestMetricsSnapshotTests.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader; - -import static org.junit.Assert.assertEquals; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.util.HashMap; -import java.util.Map; - -import org.jooq.Record; -import org.jooq.Result; -import org.junit.Test; - -public class ShardRequestMetricsSnapshotTests { - private static final String DB_URL = "jdbc:sqlite:"; - - public ShardRequestMetricsSnapshotTests() throws ClassNotFoundException { - System.setProperty("java.io.tmpdir", "/tmp"); - Class.forName("org.sqlite.JDBC"); - } - - @Test - public void testCreateRequestMetrics() throws Exception { - Connection conn = DriverManager.getConnection(DB_URL); - ShardRequestMetricsSnapshot rqMetricsSnap = new ShardRequestMetricsSnapshot(conn, 1535065195000L); - Map dimensions = new HashMap<>(); - dimensions.put(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString(), "ac-test"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString(), "1"); - dimensions.put("tid", "1"); - dimensions.put("rid", "1"); - rqMetricsSnap.putStartMetric(1535065196120L, dimensions); - rqMetricsSnap.putEndMetric(1535065196323L, dimensions); - dimensions.put("rid", "2"); - rqMetricsSnap.putStartMetric(1535065197323L, dimensions); - dimensions.put("rid", "3"); - dimensions.put("tid", "2"); - rqMetricsSnap.putStartMetric(1535065198323L, dimensions); - rqMetricsSnap.putEndMetric(1535065199923L, dimensions); - Result res = rqMetricsSnap.fetchThreadUtilizationRatio(); - Float tUtil = Float.parseFloat(res.get(0).get("tUtil").toString()); - assertEquals(0.07048611111111111f, tUtil.floatValue(), 0); - } - - @Test - public void testRollover() throws Exception { - Connection conn = DriverManager.getConnection(DB_URL); - ShardRequestMetricsSnapshot rqMetricsSnap = new ShardRequestMetricsSnapshot(conn, 1535065195000L); - Map dimensions = new HashMap<>(); - dimensions.put(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString(), "ac-test"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString(), "1"); - dimensions.put("tid", "1"); - dimensions.put("rid", "1"); - rqMetricsSnap.putStartMetric(1535065196120L, dimensions); - rqMetricsSnap.putEndMetric(1535065196323L, dimensions); - dimensions.put("rid", "2"); - rqMetricsSnap.putStartMetric(1535065197323L, dimensions); - dimensions.put("rid", "3"); - dimensions.put("tid", "2"); - rqMetricsSnap.putStartMetric(1535065198323L, dimensions); - rqMetricsSnap.putEndMetric(1535065199923L, dimensions); - ShardRequestMetricsSnapshot newSnap = new ShardRequestMetricsSnapshot(conn, 1L); - newSnap.rolloverInflightRequests(rqMetricsSnap); - } - - @Test - public void testDedup() throws Exception { - Connection conn = DriverManager.getConnection(DB_URL); - ShardRequestMetricsSnapshot rqMetricsSnap = new ShardRequestMetricsSnapshot(conn, 1535065195000L); - Map dimensions = new HashMap<>(); - dimensions.put(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString(), "ac-test"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString(), "1"); - dimensions.put("tid", "1"); - dimensions.put("rid", "1"); - rqMetricsSnap.putStartMetric(1535065191120L, dimensions); - dimensions.put("rid", "2"); - rqMetricsSnap.putStartMetric(1535065197323L, dimensions); - rqMetricsSnap.putEndMetric(1535065198323L, dimensions); - dimensions.put("rid", "3"); - dimensions.put("tid", "2"); - rqMetricsSnap.putStartMetric(1535065198323L, dimensions); - dimensions.put("rid", "4"); - dimensions.put("tid", "3"); - rqMetricsSnap.putStartMetric(1535065191323L, dimensions); - Result res = rqMetricsSnap.fetchInflightSelect().fetch(); - assertEquals(2, res.size(), 0); - } - - @Test - public void testLatestRequestNotExcluded() throws Exception { - Connection conn = DriverManager.getConnection(DB_URL); - ShardRequestMetricsSnapshot rqMetricsSnap = new ShardRequestMetricsSnapshot(conn, 1535065195000L); - Map dimensions = new HashMap<>(); - dimensions.put(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString(), "ac-test"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString(), "1"); - dimensions.put("tid", "1"); - dimensions.put("rid", "1"); - rqMetricsSnap.putStartMetric(1535065191120L, dimensions); - dimensions.put("rid", "2"); - rqMetricsSnap.putStartMetric(1535065192323L, dimensions); - rqMetricsSnap.putEndMetric(1535065198323L, dimensions); - dimensions.put("rid", "3"); - rqMetricsSnap.putStartMetric(1535065193323L, dimensions); - Result res = rqMetricsSnap.fetchInflightSelect().fetch(); - assertEquals(1, res.size(), 0); - assertEquals("3", res.get(0).get("rid")); - } - - @Test - public void testMultiOp() throws Exception { - Connection conn = DriverManager.getConnection(DB_URL); - ShardRequestMetricsSnapshot rqMetricsSnap = new ShardRequestMetricsSnapshot(conn, 1535065195000L); - Map dimensions = new HashMap<>(); - dimensions.put(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString(), "ac-test"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString(), "1"); - dimensions.put("tid", "1"); - dimensions.put("rid", "1"); - dimensions.put("operation", "shardquery"); - rqMetricsSnap.putStartMetric(1535065191120L, dimensions); - dimensions.put("tid", "2"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.OPERATION.toString(), "shardfetch"); - rqMetricsSnap.putStartMetric(1535065197323L, dimensions); - Result res = rqMetricsSnap.fetchInflightSelect().fetch(); - assertEquals(2, res.size(), 0); - } - - - @Test - public void testFetchLatency() throws Exception { - Connection conn = DriverManager.getConnection(DB_URL); - ShardRequestMetricsSnapshot rqMetricsSnap = new ShardRequestMetricsSnapshot(conn, 1535065195000L); - Map dimensions = new HashMap<>(); - dimensions.put(ShardRequestMetricsSnapshot.Fields.INDEX_NAME.toString(), "sonested"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.SHARD_ID.toString(), "0"); - dimensions.put("tid", "1"); - dimensions.put("rid", "1"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.OPERATION.toString(), "shardquery"); - rqMetricsSnap.putStartMetric(1535065191120L, dimensions); - rqMetricsSnap.putEndMetric(1535065191130L, dimensions); - dimensions.put("tid", "2"); - dimensions.put(ShardRequestMetricsSnapshot.Fields.OPERATION.toString(), "shardfetch"); - rqMetricsSnap.putStartMetric(1535065197323L, dimensions); - dimensions.put("rid", "3"); - rqMetricsSnap.putStartMetric(1535065197373L, dimensions); - dimensions.put("rid", "4"); - rqMetricsSnap.putEndMetric(1535065197388L, dimensions); - Result res = rqMetricsSnap.fetchInflightSelect().fetch(); - assertEquals(2, res.size(), 0); - } -} - diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader_writer_shared/EventLogFileHandlerTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader_writer_shared/EventLogFileHandlerTest.java new file mode 100644 index 00000000..2db1ccb1 --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader_writer_shared/EventLogFileHandlerTest.java @@ -0,0 +1,80 @@ +package com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared; + +import java.io.File; + +import org.junit.Before; +import org.junit.Test; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.HeapMetricsCollector; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.writer.EventLogQueueProcessor; + +public class EventLogFileHandlerTest { + String pathToTestMetricsDir; + + @Before + public void init() { + pathToTestMetricsDir = "/tmp/testMetrics/"; + deleteDirectory(new File(pathToTestMetricsDir)); + boolean newDir = new File(pathToTestMetricsDir).mkdir(); + } + + private boolean deleteDirectory(File directoryToBeDeleted) { + File[] allContents = directoryToBeDeleted.listFiles(); + if (allContents != null) { + for (File file : allContents) { + deleteDirectory(file); + } + } + return directoryToBeDeleted.delete(); + } + + // @After + public void cleanup() { + deleteDirectory(new File(pathToTestMetricsDir)); + } + + private long generateWriterFile(int count) { + long currTime = System.currentTimeMillis(); + HeapMetricsCollector heapMetricsCollector = new HeapMetricsCollector(); + + for (int i=0; i Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistro.elasticsearch.performanceanalyzer.rest; - -import static org.junit.Assert.assertEquals; - -import java.security.InvalidParameterException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; - -import org.junit.Test; - -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.CommonDimension; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.OSMetrics; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader.ReaderMetricsProcessor; - -@SuppressWarnings("serial") -public class QueryMetricsRequestHandlerTests { - - public QueryMetricsRequestHandlerTests() throws ClassNotFoundException { - Class.forName("org.sqlite.JDBC"); - System.setProperty("java.io.tmpdir", "/tmp"); - } - - @Test - public void testNodeJsonBuilder() throws Exception { - String rootLocation = "test_files/dev/shm"; - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - ReaderMetricsProcessor.setCurrentInstance(mp); - QueryMetricsRequestHandler qHandler = new QueryMetricsRequestHandler(); - HashMap nodeResponses = new HashMap() {{ - this.put("node1", "{'xyz':'abc'}"); - this.put("node2", "{'xyz':'abc'}"); - }}; - assertEquals("{\"node2\": {'xyz':'abc'}, \"node1\" :{'xyz':'abc'}}", - qHandler.nodeJsonBuilder(nodeResponses)); - } - - // Disabled on purpose - // @Test - public void testQueryJson() throws Exception { - String rootLocation = "build/private/test_resources/dev/shm"; - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - ReaderMetricsProcessor.setCurrentInstance(mp); - mp.processMetrics(rootLocation, 1535065139000L); - mp.processMetrics(rootLocation, 1535065169000L); - mp.processMetrics(rootLocation, 1535065199000L); - mp.processMetrics(rootLocation, 1535065229000L); - mp.processMetrics(rootLocation, 1535065259000L); - mp.processMetrics(rootLocation, 1535065289000L); - mp.processMetrics(rootLocation, 1535065319000L); - mp.processMetrics(rootLocation, 1535065349000L); - QueryMetricsRequestHandler qHandler = new QueryMetricsRequestHandler(); - String response = qHandler.collectStats(mp.getMetricsDB().getValue(), - 1234L, Arrays.asList(OSMetrics.CPU_UTILIZATION.toString()), - Arrays.asList("sum"), - Arrays.asList(CommonDimension.SHARD_ID.toString(), - CommonDimension.INDEX_NAME.toString(), - CommonDimension.OPERATION.toString()), null); - assertEquals("{\"timestamp\": 1234, \"data\": {\"fields\":[{\"name\":"+ - "\"ShardID\",\"type\":\"VARCHAR\"},{\"name\":\"IndexName\","+ - "\"type\":\"VARCHAR\"},{\"name\":\"Operation\",\"type\":"+ - "\"VARCHAR\"},{\"name\":\"CPU_Utilization\",\"type\":\"DOUBLE\""+ - "}],\"records\":[[null,null,\"GC\",0.0],[null,null,\"management\",0.0],[null,null,\"other\""+ - ",0.0256],[null,null,\"refresh\",0.0],[\"0\",\"sonested\",\"shardfetch\",0.00159186808056345],"+ - "[\"0\",\"sonested\",\"shardquery\",1.55800813191944]]}}", response); - - } - - - - @Test - public void testParseArrayParameter() throws Exception { - String rootLocation = "test_files/dev/shm"; - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - ReaderMetricsProcessor.setCurrentInstance(mp); - QueryMetricsRequestHandler qHandler = new QueryMetricsRequestHandler(); - - HashMap params = new HashMap(); - params.put("metrics", "cpu"); - - List ret = qHandler.parseArrayParam(params, "metrics", false); - assertEquals(1, ret.size()); - assertEquals("cpu", ret.get(0)); - - params.put("metrics", "cpu,rss"); - - ret = qHandler.parseArrayParam(params, "metrics", false); - assertEquals(2, ret.size()); - assertEquals("cpu", ret.get(0)); - assertEquals("rss", ret.get(1)); - } - - - - @Test - public void testParseArrayParameterOptional() throws Exception { - String rootLocation = "test_files/dev/shm"; - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - ReaderMetricsProcessor.setCurrentInstance(mp); - QueryMetricsRequestHandler qHandler = new QueryMetricsRequestHandler(); - - HashMap params = new HashMap(); - List ret = qHandler.parseArrayParam(params, "metrics", true); - assertEquals(0, ret.size()); - - params.put("metrics", ""); - ret = qHandler.parseArrayParam(params, "metrics", true); - assertEquals(0, ret.size()); - } - - - - @Test(expected = InvalidParameterException.class) - public void testParseArrayParameterNoParam() throws Exception { - String rootLocation = "test_files/dev/shm"; - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - ReaderMetricsProcessor.setCurrentInstance(mp); - QueryMetricsRequestHandler qHandler = new QueryMetricsRequestHandler(); - - HashMap params = new HashMap(); - List ret = qHandler.parseArrayParam(params, "metrics", false); - } - - @Test(expected = InvalidParameterException.class) - public void testParseArrayParameterEmptyParam() throws Exception { - String rootLocation = "test_files/dev/shm"; - ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation); - ReaderMetricsProcessor.setCurrentInstance(mp); - QueryMetricsRequestHandler qHandler = new QueryMetricsRequestHandler(); - - HashMap params = new HashMap(); - params.put("metrics", ""); - List ret = qHandler.parseArrayParam(params, "metrics", false); - } -} - diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/transport/PerformanceAnalyzerTransportChannelTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/transport/PerformanceAnalyzerTransportChannelTests.java index ab1a0798..140d74c7 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/transport/PerformanceAnalyzerTransportChannelTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/transport/PerformanceAnalyzerTransportChannelTests.java @@ -15,11 +15,16 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.transport; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; +import org.junit.Ignore; import org.junit.Test; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics; import static org.junit.Assert.assertEquals; -public class PerformanceAnalyzerTransportChannelTests { +@Ignore +public class PerformanceAnalyzerTransportChannelTests extends CustomMetricsLocationTestBase { @Test public void testShardBulkMetrics() { System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); @@ -27,9 +32,9 @@ public void testShardBulkMetrics() { PerformanceAnalyzerTransportChannel performanceanalyzerTransportChannel = new PerformanceAnalyzerTransportChannel(); performanceanalyzerTransportChannel.saveMetricValues("ABCDEF", startTimeInMills, "BulkThread", "ShardBulkId", "start"); String fetchedValue = PerformanceAnalyzerMetrics.getMetric( - PerformanceAnalyzerMetrics.sDevShmLocation + + PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)+"/threads/BulkThread/shardbulk/ShardBulkId/start"); - PerformanceAnalyzerMetrics.removeMetrics(PerformanceAnalyzerMetrics.sDevShmLocation + PerformanceAnalyzerMetrics.removeMetrics(PluginSettings.instance().getMetricsLocation() + PerformanceAnalyzerMetrics.getTimeInterval(startTimeInMills)); assertEquals("ABCDEF", fetchedValue); } diff --git a/test_files/new_format/new_format.tar.gz b/test_files/new_format/new_format.tar.gz new file mode 100644 index 00000000..2014f41d Binary files /dev/null and b/test_files/new_format/new_format.tar.gz differ diff --git a/test_files/new_format/performance-analyzer.properties b/test_files/new_format/performance-analyzer.properties new file mode 100644 index 00000000..7906acd9 --- /dev/null +++ b/test_files/new_format/performance-analyzer.properties @@ -0,0 +1,39 @@ +# ======================== Elasticsearch performance analyzer plugin config ========================= + +# NOTE: this is an example for Linux. Please modify the config accordingly if you are using it under other OS. + +# Metrics data location +# For test only +metrics-location = /tmp/new_format/ + +# Metrics deletion interval (minutes) for metrics data. +# Interval should be between 1 to 60. +metrics-deletion-interval = 1 + +# If set to true, the system cleans up the files behind it. So at any point, we should expect only 2 +# metrics-db-file-prefix-path files. If set to false, no files are cleaned up. This can be useful, if you are archiving +# the files and wouldn't like for them to be cleaned up. +cleanup-metrics-db-files = true + +# WebService exposed by App's port +webservice-listener-port = 9600 + +# Metric DB File Prefix Path location +metrics-db-file-prefix-path = /tmp/metricsdb_ + +https-enabled = false + +#Setup the correct path for certificates +certificate-file-path = specify_path + +private-key-file-path = specify_path + +# WebService bind host; default to all interfaces +#webservice-bind-host = + +# Plugin Stats Metadata file name, expected to be in the same location +plugin-stats-metadata = plugin-stats-metadata + +# Agent Stats Metadata file name, expected to be in the same location +agent-stats-metadata = agent-stats-metadata + diff --git a/test_files/old_format/old_format.tar.gz b/test_files/old_format/old_format.tar.gz new file mode 100644 index 00000000..adca1315 Binary files /dev/null and b/test_files/old_format/old_format.tar.gz differ diff --git a/test_files/old_format/performance-analyzer.properties b/test_files/old_format/performance-analyzer.properties new file mode 100644 index 00000000..af81ce29 --- /dev/null +++ b/test_files/old_format/performance-analyzer.properties @@ -0,0 +1,39 @@ +# ======================== Elasticsearch performance analyzer plugin config ========================= + +# NOTE: this is an example for Linux. Please modify the config accordingly if you are using it under other OS. + +# Metrics data location +# for tests only +metrics-location = /tmp/old_format/ + +# Metrics deletion interval (minutes) for metrics data. +# Interval should be between 1 to 60. +metrics-deletion-interval = 1 + +# If set to true, the system cleans up the files behind it. So at any point, we should expect only 2 +# metrics-db-file-prefix-path files. If set to false, no files are cleaned up. This can be useful, if you are archiving +# the files and wouldn't like for them to be cleaned up. +cleanup-metrics-db-files = true + +# WebService exposed by App's port +webservice-listener-port = 9600 + +# Metric DB File Prefix Path location +metrics-db-file-prefix-path = /tmp/metricsdb_ + +https-enabled = false + +#Setup the correct path for certificates +certificate-file-path = specify_path + +private-key-file-path = specify_path + +# WebService bind host; default to all interfaces +#webservice-bind-host = + +# Plugin Stats Metadata file name, expected to be in the same location +plugin-stats-metadata = plugin-stats-metadata + +# Agent Stats Metadata file name, expected to be in the same location +agent-stats-metadata = agent-stats-metadata +