[Backport 7.x] Add logging statements and check for build and test failure libraries

build.gradle

@@ -128,7 +128,7 @@ jacocoTestReport {
     }
 }
 
-String version = '7.0.0'
+String version = '7.0.1'
 
 task updateVersion {
     doLast {

tests/jenkins/jobs/UpdateBuildFailureIssue_Jenkinsfile.txt

@@ -23,16 +23,8 @@
             set +x
             curl -s -XGET "sample.url/opensearch-distribution-build-results/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "abc:xyz" -H "x-amz-security-token:sampleToken" -H 'Content-Type: application/json' -d "{\"_source\":[\"component\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"component_category\":\"OpenSearch\"}},{\"match_phrase\":{\"component_build_result\":\"failed\"}},{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"distribution_build_number\":\"32\"}},{\"range\":{\"build_start_time\":{\"from\":\"now-6h\",\"to\":\"now\"}}}]}}}" | jq '.'
         , returnStdout=true})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
+                        updateBuildFailureIssues.println(Failed Components: [asynchronous-search, notifications])
+                        updateBuildFailureIssues.println(Passed Components: [sql, notifications])
                   updateBuildFailureIssues.createGithubIssue({repoUrl=https://github.com/opensearch-project/notifications.git, issueTitle=[AUTOCUT] Distribution Build Failed for notifications-2.2.0, issueBody=***Build Failed Error***: **notifications failed during the distribution build for version: 2.2.0.**
                     Please see build log at www.example.com/job/build_url/32/display/redirect.
                     The failed build stage will be marked as unstable :warning: . Please see ./build.sh step for more details.
@@ -50,7 +42,6 @@ ccc --repo https://github.com/opensearch-project/notifications.git --body "***Bu
                     The failed build stage will be marked as unstable :warning: . Please see ./build.sh step for more details.
                     Checkout the [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Building-an-OpenSearch-and-OpenSearch-Dashboards-Distribution) to reproduce the failure locally.", returnStdout=true})
                   updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
                   updateBuildFailureIssues.createGithubIssue({repoUrl=https://github.com/opensearch-project/asynchronous-search.git, issueTitle=[AUTOCUT] Distribution Build Failed for asynchronous-search-2.2.0, issueBody=***Build Failed Error***: **asynchronous-search failed during the distribution build for version: 2.2.0.**
                     Please see build log at www.example.com/job/build_url/32/display/redirect.
                     The failed build stage will be marked as unstable :warning: . Please see ./build.sh step for more details.
@@ -68,8 +59,6 @@ ccc --repo https://github.com/opensearch-project/asynchronous-search.git --body
                     The failed build stage will be marked as unstable :warning: . Please see ./build.sh step for more details.
                     Checkout the [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Building-an-OpenSearch-and-OpenSearch-Dashboards-Distribution) to reproduce the failure locally.", returnStdout=true})
                   updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
                   updateBuildFailureIssues.closeGithubIssue({repoUrl=https://github.com/opensearch-project/sql.git, issueTitle=[AUTOCUT] Distribution Build Failed for sql-2.2.0, closeComment=Closing the issue as the distribution build for sql has passed for version: **2.2.0**.
                     Please see build log at www.example.com/job/build_url/32/display/redirect, label=autocut,v2.2.0})
                      closeGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})
@@ -79,5 +68,3 @@ ccc --repo https://github.com/opensearch-project/asynchronous-search.git --body
 ccc -R opensearch-project/sql --comment "Closing the issue as the distribution build for sql has passed for version: **2.2.0**.
                     Please see build log at www.example.com/job/build_url/32/display/redirect", returnStdout=true})
                   updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})
-                  updateBuildFailureIssues.sleep({time=3, unit=SECONDS})

tests/jenkins/jobs/UpdateBuildFailureIssue_without_distributionID_Jenkinsfile.txt

@@ -16,6 +16,7 @@
             set +x
             curl -s -XGET "sample.url/opensearch-distribution-build-results/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "abc:xyz" -H "x-amz-security-token:sampleToken" -H 'Content-Type: application/json' -d "{\"size\":1,\"_source\":[\"distribution_build_number\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"component_category\":\"OpenSearch\"}},{\"match_phrase\":{\"version\":\"2.2.0\"}}]}},\"sort\":[{\"build_start_time\":{\"order\":\"desc\"}}]}" | jq '.'
         , returnStdout=true})
+                        updateIntegTestFailureIssues.println(Distribution Build Number: 4891)
                         ComponentIntegTestStatus.getComponents(passed)
                            OpenSearchMetricsQuery.fetchMetrics({\"size\":50,\"_source\":[\"component\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"component_category\":\"OpenSearch\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}},{\"match_phrase\":{\"component_build_result\":\"passed\"}}]}}})
                               updateIntegTestFailureIssues.sh({script=
@@ -30,12 +31,8 @@
             set +x
             curl -s -XGET "sample.url/opensearch-integration-test-results/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "abc:xyz" -H "x-amz-security-token:sampleToken" -H 'Content-Type: application/json' -d "{\"size\":50,\"_source\":[\"component\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"component_category\":\"OpenSearch\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}},{\"match_phrase\":{\"component_build_result\":\"failed\"}}]}}}" | jq '.'
         , returnStdout=true})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
+                        updateIntegTestFailureIssues.println(Failed Components: [geospatial, k-NN])
+                        updateIntegTestFailureIssues.println(Passed Components: [cross-cluster-replication, k-NN, index-management, neural-search])
                         updateIntegTestFailureIssues.println(Integration test failed for geospatial, creating github issue)
                         ComponentIntegTestStatus.getComponentIntegTestFailedData(geospatial)
                            OpenSearchMetricsQuery.fetchMetrics({\"_source\":[\"platform\",\"architecture\",\"distribution\",\"test_report_manifest_yml\",\"integ_test_build_url\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"component\":\"geospatial\"}},{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}}]}}})
@@ -127,11 +124,6 @@ Check out test report manifest linked above for steps to reproduce, cluster and
                               closeGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/cross-cluster-replication.git -S "[AUTOCUT] Integration Test Failed for cross-cluster-replication-2.2.0 in:title" --json number --jq '.[0].number', returnStdout=true})
                               closeGithubIssue.sh({script=gh issue close bbb
 ccc -R opensearch-project/cross-cluster-replication --comment "Closing the issue as the integration tests for cross-cluster-replication passed for version: **2.2.0**.", returnStdout=true})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
                         updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
                         updateIntegTestFailureIssues.println(Integration tests passed for index-management, closing github issue)
                         updateIntegTestFailureIssues.closeGithubIssue({repoUrl=https://github.com/opensearch-project/index-management.git, issueTitle=[AUTOCUT] Integration Test Failed for index-management-2.2.0, closeComment=Closing the issue as the integration tests for index-management passed for version: **2.2.0**.})
@@ -141,6 +133,3 @@ ccc -R opensearch-project/cross-cluster-replication --comment "Closing the issue
                               closeGithubIssue.sh({script=gh issue close bbb
 ccc -R opensearch-project/index-management --comment "Closing the issue as the integration tests for index-management passed for version: **2.2.0**.", returnStdout=true})
                         updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})

tests/jenkins/jobs/UpdateIntegTestFailureIssues_Jenkinsfile.txt

@@ -9,6 +9,7 @@
                   updateIntegTestFailureIssues.string({credentialsId=jenkins-health-metrics-cluster-endpoint, variable=METRICS_HOST_URL})
                   updateIntegTestFailureIssues.withCredentials([METRICS_HOST_ACCOUNT, METRICS_HOST_URL], groovy.lang.Closure)
                      updateIntegTestFailureIssues.withAWS({role=OpenSearchJenkinsAccessRole, roleAccount=METRICS_HOST_ACCOUNT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        updateIntegTestFailureIssues.println(Distribution Build Number: 4891)
                         ComponentIntegTestStatus.getComponents(passed)
                            OpenSearchMetricsQuery.fetchMetrics({\"size\":50,\"_source\":[\"component\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"component_category\":\"OpenSearch\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}},{\"match_phrase\":{\"component_build_result\":\"passed\"}}]}}})
                               updateIntegTestFailureIssues.sh({script=
@@ -23,12 +24,8 @@
             set +x
             curl -s -XGET "sample.url/opensearch-integration-test-results/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "abc:xyz" -H "x-amz-security-token:sampleToken" -H 'Content-Type: application/json' -d "{\"size\":50,\"_source\":[\"component\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"component_category\":\"OpenSearch\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}},{\"match_phrase\":{\"component_build_result\":\"failed\"}}]}}}" | jq '.'
         , returnStdout=true})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
+                        updateIntegTestFailureIssues.println(Failed Components: [geospatial, k-NN])
+                        updateIntegTestFailureIssues.println(Passed Components: [cross-cluster-replication, k-NN, index-management, neural-search])
                         updateIntegTestFailureIssues.println(Integration test failed for geospatial, creating github issue)
                         ComponentIntegTestStatus.getComponentIntegTestFailedData(geospatial)
                            OpenSearchMetricsQuery.fetchMetrics({\"_source\":[\"platform\",\"architecture\",\"distribution\",\"test_report_manifest_yml\",\"integ_test_build_url\"],\"query\":{\"bool\":{\"filter\":[{\"match_phrase\":{\"component\":\"geospatial\"}},{\"match_phrase\":{\"version\":\"2.2.0\"}},{\"match_phrase\":{\"distribution_build_number\":\"4891\"}}]}}})
@@ -120,11 +117,6 @@ Check out test report manifest linked above for steps to reproduce, cluster and
                               closeGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/cross-cluster-replication.git -S "[AUTOCUT] Integration Test Failed for cross-cluster-replication-2.2.0 in:title" --json number --jq '.[0].number', returnStdout=true})
                               closeGithubIssue.sh({script=gh issue close bbb
 ccc -R opensearch-project/cross-cluster-replication --comment "Closing the issue as the integration tests for cross-cluster-replication passed for version: **2.2.0**.", returnStdout=true})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
                         updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
                         updateIntegTestFailureIssues.println(Integration tests passed for index-management, closing github issue)
                         updateIntegTestFailureIssues.closeGithubIssue({repoUrl=https://github.com/opensearch-project/index-management.git, issueTitle=[AUTOCUT] Integration Test Failed for index-management-2.2.0, closeComment=Closing the issue as the integration tests for index-management passed for version: **2.2.0**.})
@@ -134,6 +126,3 @@ ccc -R opensearch-project/cross-cluster-replication --comment "Closing the issue
                               closeGithubIssue.sh({script=gh issue close bbb
 ccc -R opensearch-project/index-management --comment "Closing the issue as the integration tests for index-management passed for version: **2.2.0**.", returnStdout=true})
                         updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})
-                        updateIntegTestFailureIssues.sleep({time=3, unit=SECONDS})

vars/updateBuildFailureIssues.groovy

@@ -39,14 +39,16 @@ void call(Map args = [:]) {
 
                     passedComponents = componentBuildStatus.getComponents('passed')
                     failedComponents = componentBuildStatus.getComponents('failed')
+                    println('Failed Components: '+ failedComponents)
+                    println('Passed Components: '+ passedComponents)
             }
         }
 
     failedComponents = failedComponents.unique()
     passedComponents = passedComponents.unique()
 
     for (component in inputManifest.components) {
-        if (failedComponents.contains(component.name)) {
+        if (!failedComponents.isEmpty() && failedComponents.contains(component.name)) {
             println("Component ${component.name} failed, creating github issue")
             ghIssueBody = """***Build Failed Error***: **${component.name} failed during the distribution build for version: ${currentVersion}.**
                     Please see build log at ${env.RUN_DISPLAY_URL}.
@@ -59,8 +61,9 @@ void call(Map args = [:]) {
                 label: "autocut,v${currentVersion}",
                 issueEdit: true
             )
+            sleep(time:3, unit:'SECONDS')
         }
-        if (passedComponents.contains(component.name) && !failedComponents.contains(component.name)) {
+        if (!passedComponents.isEmpty() && passedComponents.contains(component.name) && !failedComponents.contains(component.name)) {
             println("Component ${component.name} passed, closing github issue")
             ghIssueBody = """Closing the issue as the distribution build for ${component.name} has passed for version: **${currentVersion}**.
                     Please see build log at ${env.RUN_DISPLAY_URL}""".stripIndent()
@@ -70,7 +73,7 @@ void call(Map args = [:]) {
                 closeComment: ghIssueBody,
                 label: "autocut,v${currentVersion}"
             )
+            sleep(time:3, unit:'SECONDS')
         }
-        sleep(time:3, unit:'SECONDS')
     }
 }