Add a library to handle integration test failure issues (#516)

Signed-off-by: Sayali Gaikawad <[email protected]>

src/jenkins/ComponentBuildStatus.groovy

@@ -24,6 +24,7 @@ class ComponentBuildStatus {
     String buildStartTimeFrom
     String buildStartTimeTo
     def script
+    OpenSearchMetricsQuery openSearchMetricsQuery
 
     ComponentBuildStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, String distributionBuildNumber, String buildStartTimeFrom, String buildStartTimeTo, def script) {
         this.metricsUrl = metricsUrl
@@ -37,6 +38,19 @@ class ComponentBuildStatus {
         this.buildStartTimeFrom = buildStartTimeFrom
         this.buildStartTimeTo = buildStartTimeTo
         this.script = script
+        this.openSearchMetricsQuery = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script)
+    }
+
+     ComponentBuildStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, def script) {
+        this.metricsUrl = metricsUrl
+        this.awsAccessKey = awsAccessKey
+        this.awsSecretKey = awsSecretKey
+        this.awsSessionToken = awsSessionToken
+        this.indexName = indexName
+        this.product = product
+        this.version = version
+        this.script = script
+        this.openSearchMetricsQuery = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script)
     }
 
     def getQuery(String componentBuildResult) {
@@ -83,9 +97,49 @@ class ComponentBuildStatus {
         return query.replace('"', '\\"')
     }
 
+    def getLatestDistributionBuildNumberQuery() {
+        def queryMap = [
+                size : 1,
+                _source: [
+                        "distribution_build_number",
+                        ], 
+                query: [
+                        bool: [
+                                filter: [
+                                        [
+                                                match_phrase: [
+                                                        component_category: "${this.product}"
+                                                        ]
+                                                ],
+                                        [
+                                                match_phrase: [
+                                                        version: "${this.version}"
+                                                ]
+                                        ]
+                                ]
+                        ]
+                ],
+                sort : [
+                        [
+                                build_start_time: [
+                                        order: "desc"
+                                ]
+                        ]
+                ]
+        ]
+        def query = JsonOutput.toJson(queryMap)
+        return query.replace('"', '\\"')
+    }
+
     def getComponents(String componentBuildResult) {
-         def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script).fetchMetrics(getQuery(componentBuildResult))
+         def jsonResponse = this.openSearchMetricsQuery.fetchMetrics(getQuery(componentBuildResult))
          def components = jsonResponse.hits.hits.collect { it._source.component }
          return components
     }
+
+    def getLatestDistributionBuildNumber() {
+         def jsonResponse = this.openSearchMetricsQuery.fetchMetrics(getLatestDistributionBuildNumberQuery())
+         def latestDistributionBuildNumber = jsonResponse.hits.hits[0]._source.distribution_build_number
+         return latestDistributionBuildNumber
+    }
 }

src/jenkins/ComponentIntegTestStatus.groovy

@@ -0,0 +1,123 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package jenkins
+
+import groovy.json.JsonOutput
+import utils.OpenSearchMetricsQuery
+
+class ComponentIntegTestStatus {
+    String metricsUrl
+    String awsAccessKey
+    String awsSecretKey
+    String awsSessionToken
+    String indexName
+    String product
+    String version
+    String distributionBuildNumber
+    def script
+    OpenSearchMetricsQuery openSearchMetricsQuery
+
+    ComponentIntegTestStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, String distributionBuildNumber, def script) {
+        this.metricsUrl = metricsUrl
+        this.awsAccessKey = awsAccessKey
+        this.awsSecretKey = awsSecretKey
+        this.awsSessionToken = awsSessionToken
+        this.indexName = indexName
+        this.product = product
+        this.version = version
+        this.distributionBuildNumber = distributionBuildNumber
+        this.script = script
+        this.openSearchMetricsQuery = new OpenSearchMetricsQuery(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, script)
+    }
+
+    def getQuery(String componentIntegTestResult) {
+        def queryMap = [
+                size   : 50,
+                _source: [
+                        "component"
+                ],
+                query  : [
+                        bool: [
+                                filter: [
+                                        [
+                                                match_phrase: [
+                                                        version: "${this.version}"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase: [
+                                                        component_category: "${this.product}"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase: [
+                                                        distribution_build_number: "${this.distributionBuildNumber}"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase: [
+                                                        component_build_result: "${componentIntegTestResult}"
+                                                ]
+                                        ]
+                                ]
+                        ]
+                ]
+        ]
+        def query = JsonOutput.toJson(queryMap)
+        return query.replace('"', '\\"')
+    }
+
+    def componentIntegTestFailedDataQuery(String component) {
+        def queryMap = [
+                _source: [
+                        "platform",
+                        "architecture",
+                        "distribution",
+                        "test_report_manifest_yml",
+                        "integ_test_build_url"
+                ],
+                query  : [
+                        bool: [
+                                filter: [
+                                        [
+                                                match_phrase: [
+                                                        component: "${component}"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase: [
+                                                        version: "${this.version}"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase: [
+                                                        distribution_build_number: "${this.distributionBuildNumber}"
+                                                ]
+                                        ]
+                                ]
+                        ]
+                ]
+        ]
+        def query = JsonOutput.toJson(queryMap)
+        return query.replace('"', '\\"')
+    }
+
+    def getComponents(String componentBuildResult) {
+        def jsonResponse = this.openSearchMetricsQuery.fetchMetrics(getQuery(componentBuildResult))
+        def components = jsonResponse.hits.hits.collect { it._source.component }
+        return components
+    }
+
+    def getComponentIntegTestFailedData(String component) {
+        def jsonResponse = this.openSearchMetricsQuery.fetchMetrics(componentIntegTestFailedDataQuery(component))
+        return jsonResponse
+    }
+
+}

src/jenkins/CreateIntegTestMarkDownTable.groovy

@@ -0,0 +1,41 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package jenkins
+
+class CreateIntegTestMarkDownTable {
+    String version
+    ArrayList<String> tableData
+
+    CreateIntegTestMarkDownTable(String version, List<Map<String, Object>> tableData) {
+        this.version = version
+        this.tableData = tableData
+    }
+
+    def create() {
+
+        def tableHeader = """
+### Integration Test Failed for version ${version}. See the specifications below:
+
+#### Details
+
+| Platform | Distribution | Architecture | Test Report Manifest | Workflow Run |
+|----------|--------------|--------------|----------------------|--------------|
+"""
+        def tableRows = this.tableData.collect { row ->
+            "| ${row.platform} | ${row.distribution} | ${row.architecture} | ${row.test_report_manifest_yml} | ${row.integ_test_build_url}"
+        }.join("\n")
+
+        def additionalInformation = """
+\nCheck out test report manifest linked above for steps to reproduce, cluster and integration test failure logs. For additional information checkout the [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Testing-the-Distribution) and [OpenSearch Metrics Dashboard](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/21aad140-49f6-11ef-bbdd-39a9b324a5aa).
+"""
+        return tableHeader + tableRows + additionalInformation
+    }
+
+}

src/utils/OpenSearchMetricsQuery.groovy

@@ -9,7 +9,7 @@
 
 package utils
 
-import groovy.json.JsonSlurper
+import groovy.json.JsonSlurperClassic
 
 class OpenSearchMetricsQuery {
     String metricsUrl
@@ -37,6 +37,6 @@ class OpenSearchMetricsQuery {
         """,
         returnStdout: true
         ).trim()
-        return new JsonSlurper().parseText(response)
+        return new JsonSlurperClassic().parseText(response)
     }
 }