Refactor metrics query class to use any index and refactor UpdateBuil…

…dFailureIssues library with new logic

Signed-off-by: Sayali Gaikawad <[email protected]>

build.gradle

@@ -34,7 +34,7 @@ configurations.all {
     resolutionStrategy {
         force group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
         force group: 'commons-codec', name: 'commons-codec', version: '1.15'
-        force group: 'com.google.protobuf', name: 'protobuf-java', version: '3.21.7'
+        force group: 'com.google.protobuf', name: 'protobuf-java', version: '3.25.5'
         force group: 'com.google.errorprone', name: 'error_prone_annotations', version: '2.18.0'
         force group: 'org.checkerframework', name: 'checker-qual', version: '3.33.0'
         force group: 'com.google.j2objc', name: 'j2objc-annotations', version: '2.8'
@@ -51,7 +51,7 @@ sourceSets {
 
     test {
         groovy {
-            srcDirs = ['tests/jenkins', 'tests/gradlecheck', 'tests/utils']
+            srcDirs = ['tests/jenkins', 'tests/gradlecheck']
         }
     }
 
@@ -128,7 +128,7 @@ jacocoTestReport {
     }
 }
 
-String version = '6.9.2'
+String version = '7.0.0'
 
 task updateVersion {
     doLast {

src/jenkins/ComponentBuildStatus.groovy

@@ -0,0 +1,91 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package jenkins
+
+import groovy.json.JsonOutput
+import utils.OpenSearchMetricsQuery
+
+class ComponentBuildStatus {
+    String metricsUrl
+    String awsAccessKey
+    String awsSecretKey
+    String awsSessionToken
+    String indexName
+    String product 
+    String version
+    String distributionBuildNumber
+    String buildStartTimeFrom
+    String buildStartTimeTo
+    def script
+
+    ComponentBuildStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, String distributionBuildNumber, String buildStartTimeFrom, String buildStartTimeTo, def script) {
+        this.metricsUrl = metricsUrl
+        this.awsAccessKey = awsAccessKey
+        this.awsSecretKey = awsSecretKey
+        this.awsSessionToken = awsSessionToken
+        this.indexName = indexName
+        this.product = product
+        this.version = version
+        this.distributionBuildNumber = distributionBuildNumber
+        this.buildStartTimeFrom = buildStartTimeFrom
+        this.buildStartTimeTo = buildStartTimeTo
+        this.script = script
+    }
+
+    def getQuery(String componentBuildResult) {
+        def queryMap = [
+                _source: [
+                        "component",
+                        ], 
+                query: [
+                        bool: [
+                                filter: [
+                                        [
+                                                match_phrase: [
+                                                        component_category: "${this.product}"
+                                                        ]
+                                                ],
+                                        [
+                                                match_phrase: [
+                                                        component_build_result: "${componentBuildResult}"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase: [
+                                                        version: "${this.version}"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase : [
+                                                        distribution_build_number : "${this.distributionBuildNumber}"
+                                                ]
+                                        ],
+                                        [
+                                                range: [
+                                                        build_start_time: [
+                                                                from: "${this.buildStartTimeFrom}",
+                                                                to: "${this.buildStartTimeTo}"
+                                                                ]
+                                                        ]
+                                        ]
+                                ]
+                        ]
+                ]
+        ]
+        def query = JsonOutput.toJson(queryMap)
+        return query.replace('"', '\\"')
+    }
+
+    def getComponents(String componentBuildResult) {
+         def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script).fetchMetrics(getQuery(componentBuildResult))
+         def components = jsonResponse.hits.hits.collect { it._source.component }
+         return components
+    }
+}

src/utils/OpenSearchMetricsQuery.groovy

@@ -28,15 +28,14 @@ class OpenSearchMetricsQuery {
         this.script = script
     }
 
-    // Ensure the alias `gradle-check` is created targeting all the gradle-check-* indices.
     def fetchMetrics(String query) {
         def response = script.sh(
             script: """
             set -e
             set +x
             curl -s -XGET "${metricsUrl}/${indexName}/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "${awsAccessKey}:${awsSecretKey}" -H "x-amz-security-token:${awsSessionToken}" -H 'Content-Type: application/json' -d "${query}" | jq '.'
         """,
-                returnStdout: true
+        returnStdout: true
         ).trim()
         return new JsonSlurper().parseText(response)
     }

tests/jenkins/TestComponentBuildStatus.groovy

@@ -0,0 +1,124 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package jenkins
+
+import org.junit.*
+import groovy.json.JsonOutput
+import groovy.mock.interceptor.MockFor
+
+class TestComponentBuildStatus {
+
+    private ComponentBuildStatus componentBuildStatus
+    private final String metricsUrl = 'http://example.com'
+    private final String awsAccessKey = 'testAccessKey'
+    private final String awsSecretKey = 'testSecretKey'
+    private final String awsSessionToken = 'testSessionToken'
+    private final String indexName = 'opensearch-distribution-build-results-*'
+    private final String product = "OpenSearch"
+    private final String version = "2.18.0"
+    private final String distributionBuildNumber = "4891"
+    private final String buildStartTimeFrom = "now-6h"
+    private final String buildStartTimeTo = "now"
+    private def script
+
+    @Before
+    void setUp() {
+        script = new Expando()
+        script.sh = { Map args ->
+            if (args.containsKey("script")) {
+                return """
+                {
+                    "hits": {
+                    "total": {
+                    "value": 2,
+                    "relation": "eq"
+                    },
+                    "max_score": 0,
+                    "hits": [
+                    {
+                        "_index": "opensearch-distribution-build-results-09-2024",
+                        "_id": "QTVbQZIBOi-lzDIlekCk",
+                        "_score": 0,
+                        "_source": {
+                        "component": "performance-analyzer"
+                        }
+                    },
+                    {
+                        "_index": "opensearch-distribution-build-results-09-2024",
+                        "_id": "PzVbQZIBOi-lzDIlekCk",
+                        "_score": 0,
+                        "_source": {
+                        "component": "security-analytics"
+                        }
+                    }
+                    ]
+                }
+                }
+                """
+            }
+            return ""
+        }
+        componentBuildStatus = new ComponentBuildStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, product, version, distributionBuildNumber, buildStartTimeFrom, buildStartTimeTo, script)
+    }
+
+    @Test
+    void testGetQueryReturnsExpectedQuery() {
+        def expectedOutput = JsonOutput.toJson([
+                _source: [
+                        "component",
+                        ], 
+                query: [
+                        bool: [
+                                filter: [
+                                        [
+                                                match_phrase: [
+                                                        component_category: "OpenSearch"
+                                                        ]
+                                                ],
+                                        [
+                                                match_phrase: [
+                                                        component_build_result: "failed"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase: [
+                                                        version: "2.18.0"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase : [
+                                                        distribution_build_number : "4891"
+                                                ]
+                                        ],
+                                        [
+                                                range: [
+                                                        build_start_time: [
+                                                                from: "now-6h",
+                                                                to: "now"
+                                                                ]
+                                                        ]
+                                        ]
+                                ]
+                        ]
+                ]
+        ]).replace('"', '\\"')
+
+        def result = componentBuildStatus.getQuery('failed')
+        assert result == expectedOutput
+    }
+
+    @Test
+    void testComponentBuildStatusReturns() {
+        def expectedOutput = ['performance-analyzer', 'security-analytics']
+        def result = componentBuildStatus.getComponents('failed')
+
+        assert result == expectedOutput
+    }
+}

tests/jenkins/TestPublishDistributionBuildResults.groovy

@@ -106,6 +106,9 @@ class TestPublishDistributionBuildResults extends BuildPipelineTest {
                         "type": "keyword"
                     }
                 }
+            },
+            "aliases": {
+                "opensearch-distribution-build-results": {}
             }
         }'
         curl -I "METRICS_HOST_URL/test-index" --aws-sigv4 "aws:amz:us-east-1:es" --user "null:null" -H "x-amz-security-token:null" | grep -E "HTTP\\/[0-9]+(\\.[0-9]+)? 200"