From ccba02d9fdc2074a66fdd979a0a43077fb0272b3 Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Wed, 6 Apr 2022 10:02:26 -0700 Subject: [PATCH 1/5] Gradle 7, JDK related changes and OS 2.0 Signed-off-by: Sagar Upadhyaya --- .github/workflows/gradle.yml | 20 +-- DEVELOPER_GUIDE.md | 4 +- build.gradle | 140 ++++++++++-------- gradle.properties | 8 + gradle/wrapper/gradle-wrapper.properties | 2 +- .../NodeStatsFixedShardsMetricsCollector.java | 85 ----------- .../CircuitBreakerCollectorTests.java | 1 - 7 files changed, 102 insertions(+), 158 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index efc8288b..19b88140 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -13,9 +13,8 @@ jobs: strategy: matrix: java: - - 8 - 11 - - 14 + - 17 runs-on: [ubuntu-latest] name: Building PA package steps: @@ -39,15 +38,18 @@ jobs: working-directory: ./tmp/performance-analyzer run: | ./gradlew build -Dperformance-analyzer-rca.build=true \ - -Dperformance-analyzer-rca.repo="https://github.com/opensearch-project/performance-analyzer-rca.git" \ - -Dperformance-analyzer-rca.branch=main \ - -Dopensearch.version=1.3.0-SNAPSHOT + -Dperformance-analyzer-rca.repo="https://github.com/sgup432/performance-analyzer-rca.git" \ + -Dperformance-analyzer-rca.branch=jdk_gradle7_OS_2.0 - name: Assemble PA jar for BWC tests working-directory: ./tmp/performance-analyzer run: | - ./gradlew assemble -Dopensearch.version=1.3.0-SNAPSHOT - mkdir -p ./src/test/resources/org/opensearch/performanceanalyzer/bwc/1.3.0.0-SNAPSHOT - cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/performanceanalyzer/bwc/1.3.0.0-SNAPSHOT + ./gradlew assemble + version=`./gradlew properties -q | grep "opensearch_version:" | awk '{print $2}'` + IFS='-' read -r -a version_array <<< "$version" + plugin_version="${version_array[0]}.0"; for entry in ${version_array[@]:1}; do plugin_version+="-$entry"; done + echo $plugin_version + mkdir -p ./src/test/resources/org/opensearch/performanceanalyzer/bwc/$plugin_version + cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/performanceanalyzer/bwc/$plugin_version - name: Generate Jacoco coverage report working-directory: ./tmp/performance-analyzer run: ./gradlew jacocoTestReport @@ -58,7 +60,7 @@ jobs: run: bash <(curl -s https://codecov.io/bash) -f ./build/reports/jacoco/test/jacocoTestReport.xml - name: Run Integration Tests working-directory: ./tmp/performance-analyzer - run: ./gradlew integTest -Dtests.enableIT -Dtests.useDockerCluster -Dopensearch.version=1.3.0-SNAPSHOT + run: ./gradlew integTest -Dtests.enableIT -Dtests.useDockerCluster - name: Run PerformanceAnalzyer Backwards Compatibility Tests working-directory: ./tmp/performance-analyzer run: ./gradlew bwcTestSuite -Dtests.security.manager=false diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 23fc6e2d..665f3928 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1,7 +1,7 @@ - [Developer Guide](#developer-guide) - [Forking and Cloning](#forking-and-cloning) - [Install Prerequisites](#install-prerequisites) - - [JDK 11](#jdk-11) + - [JDK](#jdk) - [Building](#building) - [Using IntelliJ IDEA](#using-intellij-idea) - [Submitting Changes](#submitting-changes) @@ -18,7 +18,7 @@ Fork this repository on GitHub, and clone locally with `git clone`. #### JDK 11 -OpenSearch components build using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. +OpenSearch components build using Java 11 at a minimum and supports JDK 11, 17. This means you must have a JDK of supported version installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. ### Building diff --git a/build.gradle b/build.gradle index 86dd785e..686ccb13 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,19 @@ buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "1.3.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.0.0-alpha1-SNAPSHOT") + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + buildVersionQualifier = System.getProperty("build.version_qualifier", "alpha1") + + // 2.0.0-alpha1-SNAPSHOT -> 2.0.0.0-alpha1-SNAPSHOT + version_tokens = opensearch_version.tokenize('-') + opensearch_build = version_tokens[0] + '.0' + if (buildVersionQualifier) { + opensearch_build += "-${buildVersionQualifier}" + } + if (isSnapshot) { + opensearch_build += "-SNAPSHOT" + } } // Used to resolve build file dependencies @@ -26,7 +38,7 @@ buildscript { plugins { id 'java' id 'nebula.ospackage' version "8.3.0" - id 'com.github.spotbugs' version '4.6.0' + id 'com.github.spotbugs' version '5.0.0' id 'jacoco' id 'com.diffplug.spotless' version '5.11.0' id 'checkstyle' @@ -48,8 +60,6 @@ spotbugsTest { } ext { - isSnapshot = "true" == System.getProperty("build.snapshot", "true") - // The RCA branch that will be built. Default branch is main. rcaProjectRepo = System.getProperty("performance-analyzer-rca.repo", "https://github.com/opensearch-project/performance-analyzer-rca.git") rcaProjectBranch = System.getProperty("performance-analyzer-rca.branch", "main") @@ -63,10 +73,7 @@ ext { allprojects { group = "org.opensearch" - version = opensearch_version - '-SNAPSHOT' + '.0' - if (isSnapshot) { - version += "-SNAPSHOT" - } + version = opensearch_build } apply plugin: 'opensearch.opensearchplugin' @@ -77,7 +84,7 @@ noticeFile = rootProject.file('NOTICE.txt') spotless { java { licenseHeaderFile(file('license-header')) - googleJavaFormat().aosp() + googleJavaFormat('1.12.0').aosp() importOrder() removeUnusedImports() trimTrailingWhitespace() @@ -105,22 +112,14 @@ opensearchplugin { classname 'org.opensearch.performanceanalyzer.PerformanceAnalyzerPlugin' } -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_11 +targetCompatibility = JavaVersion.VERSION_11 compileJava { dependsOn spotlessApply - JavaVersion targetVersion = JavaVersion.toVersion(targetCompatibility); - if (targetVersion.isJava9Compatible()) { - options.compilerArgs += ["--add-exports", "jdk.attach/sun.tools.attach=ALL-UNNAMED"] - } } javadoc { - JavaVersion targetVersion = JavaVersion.toVersion(targetCompatibility); - if (targetVersion.isJava9Compatible()) { - options.addStringOption("-add-exports", "jdk.attach/sun.tools.attach=ALL-UNNAMED") - } } project.afterEvaluate { @@ -152,7 +151,7 @@ tasks.withType(JavaCompile) { } jacoco { - toolVersion = "0.8.5" + toolVersion = "0.8.7" } jacocoTestReport { @@ -205,7 +204,7 @@ checkstyleTest.enabled = false dependencies { if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { - compile files("${System.properties['java.home']}/../lib/tools.jar") + implementation files("${System.properties['java.home']}/../lib/tools.jar") } def jacksonVersion = "2.12.6" @@ -226,88 +225,91 @@ dependencies { force "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" force "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" force "com.fasterxml.jackson.module:jackson-module-paranamer:${jacksonVersion}" + force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${jacksonVersion}" + force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonVersion}" + force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jacksonVersion}" } } - compile('com.google.guava:guava:30.1-jre') { + implementation('com.google.guava:guava:30.1-jre') { force = 'true' } - compile 'org.jooq:jooq:3.10.8' - compile 'org.apache.commons:commons-lang3:3.9' - compile 'org.bouncycastle:bcprov-jdk15on:1.70' - compile 'org.bouncycastle:bcpkix-jdk15on:1.70' - compile "org.opensearch:performanceanalyzer-rca:${version}" - compile "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}" - compile "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" - compile "com.fasterxml.jackson.module:jackson-module-paranamer:${jacksonVersion}" - compile(group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.1') - compile(group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.1') - compile(group: 'com.google.errorprone', name: 'error_prone_annotations', version: '2.9.0') { + implementation 'org.jooq:jooq:3.10.8' + implementation 'org.apache.commons:commons-lang3:3.9' + implementation 'org.bouncycastle:bcprov-jdk15on:1.70' + implementation 'org.bouncycastle:bcpkix-jdk15on:1.70' + implementation "org.opensearch:performanceanalyzer-rca:${version}" + implementation "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}" + implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" + implementation "com.fasterxml.jackson.module:jackson-module-paranamer:${jacksonVersion}" + implementation(group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.1') + implementation(group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.1') + implementation(group: 'com.google.errorprone', name: 'error_prone_annotations', version: '2.9.0') { force = 'true' } - compile(group: 'com.google.protobuf', name:'protobuf-java', version: '3.19.2') { + implementation(group: 'com.google.protobuf', name:'protobuf-java', version: '3.19.2') { force = 'true' } - compile("io.netty:netty-buffer:${nettyVersion}") { + implementation("io.netty:netty-buffer:${nettyVersion}") { force = 'true' } - compile("io.netty:netty-codec:${nettyVersion}") { + implementation("io.netty:netty-codec:${nettyVersion}") { force = 'true' } - compile("io.netty:netty-codec-http:${nettyVersion}") { + implementation("io.netty:netty-codec-http:${nettyVersion}") { force = 'true' } - compile("io.netty:netty-common:${nettyVersion}") { + implementation("io.netty:netty-common:${nettyVersion}") { force = 'true' } - compile("io.netty:netty-handler:${nettyVersion}") { + implementation("io.netty:netty-handler:${nettyVersion}") { force = 'true' } - compile("io.netty:netty-resolver:${nettyVersion}") { + implementation("io.netty:netty-resolver:${nettyVersion}") { force = 'true' } - compile("io.netty:netty-transport:${nettyVersion}") { + implementation("io.netty:netty-transport:${nettyVersion}") { force = 'true' } - compile("io.netty:netty-codec-http2:${nettyVersion}") { + implementation("io.netty:netty-codec-http2:${nettyVersion}") { force = 'true' } - compile("io.netty:netty-codec-socks:${nettyVersion}") { + implementation("io.netty:netty-codec-socks:${nettyVersion}") { force = 'true' } - compile("io.netty:netty-handler-proxy:${nettyVersion}") { + implementation("io.netty:netty-handler-proxy:${nettyVersion}") { force = 'true' } // JDK9+ has to run powermock 2+. https://github.com/powermock/powermock/issues/888 - testCompile group: 'org.powermock', name: 'powermock-api-mockito2', version: '2.0.0' - testCompile(group: 'org.powermock', name: 'powermock-module-junit4', version: '2.0.0') { + testImplementation group: 'org.powermock', name: 'powermock-api-mockito2', version: '2.0.0' + testImplementation(group: 'org.powermock', name: 'powermock-module-junit4', version: '2.0.0') { exclude(group: 'org.hamcrest', module: 'hamcrest-core') } - testCompile(group: 'org.mockito', name: 'mockito-core', version: '2.23.0') { + testImplementation(group: 'org.mockito', name: 'mockito-core', version: '2.23.0') { force = 'true' } // Dependency to mock final classes. testImplementation group: 'org.mockito', name: 'mockito-inline', version: '2.13.0' - testCompile group: 'org.powermock', name: 'powermock-core', version: '2.0.0' - testCompile group: 'org.powermock', name: 'powermock-api-support', version: '2.0.0' - testCompile(group: 'org.powermock', name: 'powermock-module-junit4-common', version: '2.0.0') { + testImplementation group: 'org.powermock', name: 'powermock-core', version: '2.0.0' + testImplementation group: 'org.powermock', name: 'powermock-api-support', version: '2.0.0' + testImplementation(group: 'org.powermock', name: 'powermock-module-junit4-common', version: '2.0.0') { exclude(group: 'org.hamcrest', module: 'hamcrest-core') } - testCompile group: 'org.javassist', name: 'javassist', version: '3.24.0-GA' - testCompile group: 'org.powermock', name: 'powermock-reflect', version: '2.0.0' + testImplementation group: 'org.javassist', name: 'javassist', version: '3.24.0-GA' + testImplementation group: 'org.powermock', name: 'powermock-reflect', version: '2.0.0' //minimum byte-buddy version to be compatible with mockito-core 2.23.0 is 1.9.7+. https://github.com/mockito/mockito/issues/1606 - testCompile(group: 'net.bytebuddy', name: 'byte-buddy', version: '1.9.7') { + testImplementation(group: 'net.bytebuddy', name: 'byte-buddy', version: '1.9.7') { force = 'true' } - testCompile(group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.9.7') { + testImplementation(group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.9.7') { force = 'true' } - testCompile(group: 'org.objenesis', name: 'objenesis', version: '3.0.1') { + testImplementation(group: 'org.objenesis', name: 'objenesis', version: '3.0.1') { force = 'true' } - testCompile group: 'com.google.code.gson', name: 'gson', version: '2.8.9' + testImplementation group: 'com.google.code.gson', name: 'gson', version: '2.8.9' } dependencyLicenses { @@ -363,11 +365,21 @@ task buildRca() { doLast { exec { workingDir("$rcaProjectDir") - commandLine './gradlew', 'build', '-x', 'test', "-Dopensearch.version=${opensearch_version}", "-Dbuild.snapshot=${isSnapshot}" + if (buildVersionQualifier == null || buildVersionQualifier == '' || buildVersionQualifier == 'null') { + commandLine './gradlew', 'build', '-x', 'test', "-Dopensearch.version=${opensearch_version}", "-Dbuild.snapshot=${isSnapshot}" + } + else { + commandLine './gradlew', 'build', '-x', 'test', "-Dopensearch.version=${opensearch_version}", "-Dbuild.snapshot=${isSnapshot}", "-Dbuild.version_qualifier=${buildVersionQualifier}" + } } exec { workingDir("$rcaProjectDir") - commandLine './gradlew', 'publishToMavenLocal', "-Dopensearch.version=${opensearch_version}", "-Dbuild.snapshot=${isSnapshot}" + if (buildVersionQualifier == null || buildVersionQualifier == '' || buildVersionQualifier == 'null') { + commandLine './gradlew', 'publishToMavenLocal', "-Dopensearch.version=${opensearch_version}", "-Dbuild.snapshot=${isSnapshot}" + } + else { + commandLine './gradlew', 'publishToMavenLocal', "-Dopensearch.version=${opensearch_version}", "-Dbuild.snapshot=${isSnapshot}", "-Dbuild.version_qualifier=${buildVersionQualifier}" + } } exec { def licenseDir = "$projectDir/licenses" @@ -394,6 +406,14 @@ task unpackRca(type: Copy) { } } +tasks.withType(Test) { + jvmArgs('--add-opens=java.base/sun.security.jca=ALL-UNNAMED') + jvmArgs('--add-opens=java.base/java.util.concurrent=ALL-UNNAMED') + jvmArgs('--add-opens=java.base/java.io=ALL-UNNAMED') + jvmArgs('--add-opens=java.base/java.nio.file=ALL-UNNAMED') +} + + bundlePlugin { dependsOn 'unpackRca' from("$rcaArtifactsDir/pa_config") { @@ -515,7 +535,7 @@ String bwcFilePath = "src/test/resources/org/opensearch/performanceanalyzer/bwc/ testClusters { "${baseName}$i" { testDistribution = "ARCHIVE" - versions = ["7.10.2","1.3.0-SNAPSHOT"] + versions = ["7.10.2", opensearch_version] numberOfNodes = 3 plugin(provider(new Callable(){ @Override @@ -655,7 +675,7 @@ afterEvaluate { ospackage { packageName = "opensearch-performance-analyzer" release = isSnapshot ? "0.1" : '1' - version = "${project.version}" - "-SNAPSHOT" + version = "${project.version}" into '/usr/share/opensearch/plugins' from(zipTree(bundlePlugin.archivePath)) { diff --git a/gradle.properties b/gradle.properties index 431c9c9f..43e64367 100644 --- a/gradle.properties +++ b/gradle.properties @@ -28,3 +28,11 @@ systemProp.tests.https=false systemProp.tests.user=admin # The password of the user specified above systemProp.tests.password=admin + +# Below were added to fix build issue. Refer - https://github.com/diffplug/spotless/issues/834 +org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ + --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ + --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \ + --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \ + --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED \ + --add-exports jdk.attach/sun.tools.attach=ALL-UNNAMED diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 33682bbb..e750102e 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollector.java index 76218859..6187d57a 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollector.java +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollector.java @@ -106,42 +106,6 @@ private void populateCurrentShards() { put( ShardStatsValue.SEGMENTS_TOTAL.toString(), (shardStats) -> shardStats.getStats().getSegments().getCount()); - put( - ShardStatsValue.SEGMENTS_MEMORY.toString(), - (shardStats) -> shardStats.getStats().getSegments().getMemoryInBytes()); - put( - ShardStatsValue.TERMS_MEMORY.toString(), - (shardStats) -> - shardStats.getStats().getSegments().getTermsMemoryInBytes()); - put( - ShardStatsValue.STORED_FIELDS_MEMORY.toString(), - (shardStats) -> - shardStats - .getStats() - .getSegments() - .getStoredFieldsMemoryInBytes()); - put( - ShardStatsValue.TERM_VECTOR_MEMORY.toString(), - (shardStats) -> - shardStats - .getStats() - .getSegments() - .getTermVectorsMemoryInBytes()); - put( - ShardStatsValue.NORMS_MEMORY.toString(), - (shardStats) -> - shardStats.getStats().getSegments().getNormsMemoryInBytes()); - put( - ShardStatsValue.POINTS_MEMORY.toString(), - (shardStats) -> - shardStats.getStats().getSegments().getPointsMemoryInBytes()); - put( - ShardStatsValue.DOC_VALUES_MEMORY.toString(), - (shardStats) -> - shardStats - .getStats() - .getSegments() - .getDocValuesMemoryInBytes()); put( ShardStatsValue.INDEX_WRITER_MEMORY.toString(), (shardStats) -> @@ -273,13 +237,6 @@ public class NodeStatsMetricsFixedShardsPerCollectionStatus extends MetricStatus private final long mergeCurrent; private final long indexBufferBytes; private final long segmentCount; - private final long segmentsMemory; - private final long termsMemory; - private final long storedFieldsMemory; - private final long termVectorsMemory; - private final long normsMemory; - private final long pointsMemory; - private final long docValuesMemory; private final long indexWriterMemory; private final long versionMapMemory; private final long bitsetMemory; @@ -299,13 +256,6 @@ public NodeStatsMetricsFixedShardsPerCollectionStatus(ShardStats shardStats) { this.mergeCurrent = calculate(ShardStatsValue.MERGE_CURRENT_EVENT); this.indexBufferBytes = calculate(ShardStatsValue.INDEXING_BUFFER); this.segmentCount = calculate(ShardStatsValue.SEGMENTS_TOTAL); - this.segmentsMemory = calculate(ShardStatsValue.SEGMENTS_MEMORY); - this.termsMemory = calculate(ShardStatsValue.TERMS_MEMORY); - this.storedFieldsMemory = calculate(ShardStatsValue.STORED_FIELDS_MEMORY); - this.termVectorsMemory = calculate(ShardStatsValue.TERMS_MEMORY); - this.normsMemory = calculate(ShardStatsValue.NORMS_MEMORY); - this.pointsMemory = calculate(ShardStatsValue.POINTS_MEMORY); - this.docValuesMemory = calculate(ShardStatsValue.DOC_VALUES_MEMORY); this.indexWriterMemory = calculate(ShardStatsValue.INDEX_WRITER_MEMORY); this.versionMapMemory = calculate(ShardStatsValue.VERSION_MAP_MEMORY); this.bitsetMemory = calculate(ShardStatsValue.BITSET_MEMORY); @@ -371,41 +321,6 @@ public long getSegmentCount() { return segmentCount; } - @JsonProperty(ShardStatsValue.Constants.SEGMENTS_MEMORY_VALUE) - public long getSegmentsMemory() { - return segmentsMemory; - } - - @JsonProperty(ShardStatsValue.Constants.TERMS_MEMORY_VALUE) - public long getTermsMemory() { - return termsMemory; - } - - @JsonProperty(ShardStatsValue.Constants.STORED_FIELDS_MEMORY_VALUE) - public long getStoredFieldsMemory() { - return storedFieldsMemory; - } - - @JsonProperty(ShardStatsValue.Constants.TERM_VECTOR_MEMORY_VALUE) - public long getTermVectorsMemory() { - return termVectorsMemory; - } - - @JsonProperty(ShardStatsValue.Constants.NORMS_MEMORY_VALUE) - public long getNormsMemory() { - return normsMemory; - } - - @JsonProperty(ShardStatsValue.Constants.POINTS_MEMORY_VALUE) - public long getPointsMemory() { - return pointsMemory; - } - - @JsonProperty(ShardStatsValue.Constants.DOC_VALUES_MEMORY_VALUE) - public long getDocValuesMemory() { - return docValuesMemory; - } - @JsonProperty(ShardStatsValue.Constants.INDEX_WRITER_MEMORY_VALUE) public long getIndexWriterMemory() { return indexWriterMemory; diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java index 67429282..1356a669 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java @@ -76,7 +76,6 @@ public void testCollectMetrics() throws IOException { assertEquals(CircuitBreaker.REQUEST, metrics.get(0).getType()); assertEquals(CircuitBreaker.FIELDDATA, metrics.get(1).getType()); assertEquals(CircuitBreaker.IN_FLIGHT_REQUESTS, metrics.get(2).getType()); - assertEquals(CircuitBreaker.ACCOUNTING, metrics.get(3).getType()); assertEquals(CircuitBreaker.PARENT, metrics.get(4).getType()); } From 64b8d8130744fd0c4a61bee7c47c237c8af14feb Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Wed, 6 Apr 2022 12:31:24 -0700 Subject: [PATCH 2/5] Changing logic for clone git repo to make it compatible with gradle 7 Signed-off-by: Sagar Upadhyaya --- build.gradle | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index 686ccb13..5fa16e6e 100644 --- a/build.gradle +++ b/build.gradle @@ -42,6 +42,7 @@ plugins { id 'jacoco' id 'com.diffplug.spotless' version '5.11.0' id 'checkstyle' + id 'org.ajoberstar.grgit' version '5.0.0' } spotbugsMain { @@ -327,7 +328,7 @@ gradle.startParameter.excludedTaskNames += [ "forbiddenApisMain", "testingConventions"] import java.util.concurrent.Callable -import org.ajoberstar.gradle.git.tasks.GitClone +import org.ajoberstar.grgit.Grgit import org.opensearch.gradle.test.RestIntegTestTask import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask @@ -342,16 +343,21 @@ static def propEnabled(property) { * cloneRcaGitRepo clones the performance-analyzer-rca repo if the -Dtests.enableIT=true flag is passed * to the Gradle JVM */ -task cloneRcaGitRepo(type: GitClone) { +task cloneRcaGitRepo() { def destination = file(rcaProjectDir) - uri = rcaProjectRepo - branch = rcaProjectBranch - destinationPath = destination - bare = false - enabled = cloneRcaProject && !destination.exists() // to clone only once - + def uri = rcaProjectRepo + def branch = rcaProjectBranch doFirst { - logger.info("Cloning performance-analyzer-rca into ${rcaProjectDir} from ${rcaProjectRepo}#${rcaProjectBranch}") + println "Cloning performance-analyzer-rca into ${rcaProjectDir} from ${rcaProjectRepo}#${rcaProjectBranch}" + } + doLast { + if (destination.exists() && destination.isDirectory()) { // If directory exists, use this instead. + def grgit = Grgit.open(dir: destination.toString()) + } else { // Otherwise pull it from git. + def grgit = Grgit.clone(dir: destination.toString(), uri: uri) + grgit.fetch(remote: 'origin') + grgit.checkout(branch: branch) + } } } From 606eff176d11119e4425bf28bfbaa29893928670 Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Wed, 6 Apr 2022 13:39:23 -0700 Subject: [PATCH 3/5] Fixing unit tests to remove reference of deleted fields in OS 2.0 Signed-off-by: Sagar Upadhyaya --- .../collectors/CircuitBreakerCollectorTests.java | 9 +++++---- .../NodeStatsFixedShardsMetricsCollectorTests.java | 7 ------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java index 1356a669..7e42df16 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/CircuitBreakerCollectorTests.java @@ -10,6 +10,7 @@ import com.fasterxml.jackson.module.paranamer.ParanamerModule; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.junit.After; import org.junit.Before; @@ -72,11 +73,11 @@ public void testCollectMetrics() throws IOException { createIndex(TEST_INDEX); collector.collectMetrics(startTimeInMills); List metrics = readMetrics(); - assertEquals(5, metrics.size()); + assertEquals(4, metrics.size()); assertEquals(CircuitBreaker.REQUEST, metrics.get(0).getType()); assertEquals(CircuitBreaker.FIELDDATA, metrics.get(1).getType()); assertEquals(CircuitBreaker.IN_FLIGHT_REQUESTS, metrics.get(2).getType()); - assertEquals(CircuitBreaker.PARENT, metrics.get(4).getType()); + assertEquals(CircuitBreaker.PARENT, metrics.get(3).getType()); } private List readMetrics() throws IOException { @@ -84,9 +85,9 @@ private List readMetrics() throws assert metrics.size() == 1; ObjectMapper objectMapper = new ObjectMapper().registerModule(new ParanamerModule()); String[] jsonStrs = metrics.get(0).value.split("\n"); - assert jsonStrs.length == 6; + assert jsonStrs.length == 5; List list = new ArrayList<>(); - for (int i = 1; i < 6; i++) { + for (int i = 1; i < 5; i++) { list.add( objectMapper.readValue( jsonStrs[i], CircuitBreakerCollector.CircuitBreakerStatus.class)); diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollectorTests.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollectorTests.java index 87ed1c9a..dad12aa3 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollectorTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/NodeStatsFixedShardsMetricsCollectorTests.java @@ -131,13 +131,6 @@ public void testCollectMetrics() { assertTrue(jsonStr.contains(ShardStatsValue.Constants.MERGE_CURRENT_VALUE)); assertTrue(jsonStr.contains(ShardStatsValue.Constants.INDEX_BUFFER_BYTES_VALUE)); assertTrue(jsonStr.contains(ShardStatsValue.Constants.SEGMENTS_COUNT_VALUE)); - assertTrue(jsonStr.contains(ShardStatsValue.Constants.SEGMENTS_MEMORY_VALUE)); - assertTrue(jsonStr.contains(ShardStatsValue.Constants.TERMS_MEMORY_VALUE)); - assertTrue(jsonStr.contains(ShardStatsValue.Constants.STORED_FIELDS_MEMORY_VALUE)); - assertTrue(jsonStr.contains(ShardStatsValue.Constants.TERM_VECTOR_MEMORY_VALUE)); - assertTrue(jsonStr.contains(ShardStatsValue.Constants.NORMS_MEMORY_VALUE)); - assertTrue(jsonStr.contains(ShardStatsValue.Constants.POINTS_MEMORY_VALUE)); - assertTrue(jsonStr.contains(ShardStatsValue.Constants.DOC_VALUES_MEMORY_VALUE)); assertTrue(jsonStr.contains(ShardStatsValue.Constants.INDEX_WRITER_MEMORY_VALUE)); assertTrue(jsonStr.contains(ShardStatsValue.Constants.VERSION_MAP_MEMORY_VALUE)); assertTrue(jsonStr.contains(ShardStatsValue.Constants.BITSET_MEMORY_VALUE)); From 8a9f1fa4bd9a5a1567c3e7c0c041eb96f6a2496b Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Wed, 6 Apr 2022 13:51:38 -0700 Subject: [PATCH 4/5] Fixing apache Conflict depedency issue Signed-off-by: Sagar Upadhyaya --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 5fa16e6e..e57a4945 100644 --- a/build.gradle +++ b/build.gradle @@ -229,6 +229,7 @@ dependencies { force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${jacksonVersion}" force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonVersion}" force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jacksonVersion}" + force "org.apache.commons:commons-lang3:3.12.0" } } From f113026945ed5e8e60ae98cc06ae0c3e093d0469 Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Wed, 6 Apr 2022 15:40:54 -0700 Subject: [PATCH 5/5] Reverting temporary changes to point to local RCA repo Signed-off-by: Sagar Upadhyaya --- .github/workflows/gradle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 19b88140..ea6d3430 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -38,8 +38,8 @@ jobs: working-directory: ./tmp/performance-analyzer run: | ./gradlew build -Dperformance-analyzer-rca.build=true \ - -Dperformance-analyzer-rca.repo="https://github.com/sgup432/performance-analyzer-rca.git" \ - -Dperformance-analyzer-rca.branch=jdk_gradle7_OS_2.0 + -Dperformance-analyzer-rca.repo="https://github.com/opensearch-project/performance-analyzer-rca.git" \ + -Dperformance-analyzer-rca.branch=main - name: Assemble PA jar for BWC tests working-directory: ./tmp/performance-analyzer run: |