From 9a7a3235969852adc06223307a227fba0e9826a6 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Wed, 26 Jun 2024 16:08:18 -0700 Subject: [PATCH] Mechanism to close the created Gradle Check AUTOCUT flaky test issues (#448) Signed-off-by: Prudhvi Godithi --- .codecov.yml | 4 +- .github/workflows/groovy-tests.yml | 3 +- README.md | 7 +- build.gradle | 2 +- .../FetchPostMergeFailedTestClass.groovy | 33 ++- src/gradlecheck/MarkdownComparator.groovy | 33 +++ src/gradlecheck/OpenSearchMetricsQuery.groovy | 4 +- src/gradlecheck/ParseMarkDownTable.groovy | 32 +++ .../CreateMarkDownTableTest.groovy | 3 - .../FetchPostMergeFailedTestClassTest.groovy | 32 ++- .../gradlecheck/MarkdownComparatorTest.groovy | 208 ++++++++++++++++++ .../gradlecheck/ParseMarkDownTableTest.groovy | 45 ++++ ...TestGradleCheckFlakyTestGitHubIssue.groovy | 58 +++++ ...FlakyTestGitHubIssueCreate_Jenkinsfile.txt | 12 + ...ckFlakyTestGitHubIssueEdit_Jenkinsfile.txt | 13 ++ ...radleCheckFlakyTestGitHubIssue_Jenkinsfile | 27 +++ vars/gradleCheckFlakyTestDetector.groovy | 6 +- vars/gradleCheckFlakyTestGitHubIssue.groovy | 88 ++++++++ 18 files changed, 586 insertions(+), 24 deletions(-) create mode 100644 src/gradlecheck/MarkdownComparator.groovy create mode 100644 src/gradlecheck/ParseMarkDownTable.groovy create mode 100644 tests/gradlecheck/MarkdownComparatorTest.groovy create mode 100644 tests/gradlecheck/ParseMarkDownTableTest.groovy create mode 100644 tests/jenkins/TestGradleCheckFlakyTestGitHubIssue.groovy create mode 100644 tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueCreate_Jenkinsfile.txt create mode 100644 tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueEdit_Jenkinsfile.txt create mode 100644 tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile create mode 100644 vars/gradleCheckFlakyTestGitHubIssue.groovy diff --git a/.codecov.yml b/.codecov.yml index ccd67751e..272de2dfe 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -6,5 +6,5 @@ coverage: status: project: default: - target: auto - threshold: 0.2% + target: 70% + threshold: 3% diff --git a/.github/workflows/groovy-tests.yml b/.github/workflows/groovy-tests.yml index 8137989e0..bc8bacb6a 100644 --- a/.github/workflows/groovy-tests.yml +++ b/.github/workflows/groovy-tests.yml @@ -22,6 +22,7 @@ jobs: run: | ./gradlew test --info - name: Upload Coverage Report - uses: codecov/codecov-action@v3.1.0 + uses: codecov/codecov-action@v4 with: + token: ${{ secrets.CODECOV_TOKEN }} files: ./build/reports/jacoco/test/jacocoTestReport.xml diff --git a/README.md b/README.md index 23a3a5693..d19ffc6a4 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,13 @@ lib = library(identifier: 'jenkins@', retriever: modernSCM([ | [publishToRubyGems.groovy](./vars/publishToRubyGems.groovy) | A library to publish gems to [rubygems.org](https://rubygems.org/) with [opensearchproject](https://rubygems.org/profiles/opensearchproject) as the owner. Please note that this library expects the gems to be pre-signed. You can use [PublishToRubyGemsLibTester](./tests/jenkins/lib-testers/PublishToRubyGemsLibTester.groovy) to add tests in your repository. See how to use the lib in your [jenkinsFile](./tests/jenkins/jobs/PublishToRubyGems_JenkinsFile). | | [publishToMaven.groovy](./vars/publishToMaven.groovy) | A library to sign and deploy opensearch maven artifacts to sonatype staging repository, it also has an optional parameter `autoPublish` to auto-release artifacts from staging repo to prod without manual intervention. You can use [PublishToMavenLibTester](./tests/jenkins/lib-testers/PublishToMavenLibTester.groovy) to add tests in your repository. See how to use the lib in your [jenkinsFile](./tests/jenkins/jobs/PublishToMaven_JenkinsFile). | | [publishToNuget.groovy](./vars/publishToNuget.groovy) | A library to build, sign and publish dotnet artifacts to [Nuget Gallery](https://www.nuget.org/). Please check if the [default docker](https://github.com/opensearch-project/opensearch-build/blob/main/docker/ci/dockerfiles/current/release.centos.clients.x64.arm64.dockerfile) file contains the required dotnet sdk. You can use [PublishToNugetLibTester](./tests/jenkins/lib-testers/PublishToNugetLibTester.groovy) to add tests in your repository. See how to use the lib in your [jenkinsFile](./tests/jenkins/jobs/PublishToNuget_Jenkinsfile). -| [publishToArtifactsProdBucket.groovy](./vars/publishToArtifactsProdBucket.groovy) | This library signs and uploads the artifacts to production S3 bucket which points to artifacts.opensearch.org. Please make sure the role that you use to upload exists and has the right permission. For artifacts of different types like macos, linux and windows, call this lib for each artifact with different signing parameters. You can use [PublishToArtifactsProdBucketLibTester](./tests/jenkins/lib-testers/PublishToArtifactsProdBucketLibTester.groovy) to add tests in your repository. See how to use the lib in your [jenkinsFile](./tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile). +| [publishToArtifactsProdBucket.groovy](./vars/publishToArtifactsProdBucket.groovy) | This library signs and uploads the artifacts to production S3 bucket which points to artifacts.opensearch.org. Please make sure the role that you use to upload exists and has the right permission. For artifacts of different types like macos, linux and windows, call this lib for each artifact with different signing parameters. You can use [PublishToArtifactsProdBucketLibTester](./tests/jenkins/lib-testers/PublishToArtifactsProdBucketLibTester.groovy) to add tests in your repository. See how to use the lib in your [jenkinsFile](./tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile). | [buildMessage.groovy](./vars/buildMessage.groovy) | This library that can parse the jenkins build log based on the user defined input query string. -| [closeBuildSuccessGithubIssue.groovy](./vars/closeBuildSuccessGithubIssue.groovy) | This library that identifies the successfully built components and closes the created [AUTOCUT] issues. +| [closeBuildSuccessGithubIssue.groovy](./vars/closeBuildSuccessGithubIssue.groovy) | This library that identifies the successfully built components and closes the created [AUTOCUT] issues. | [createGithubIssue.groovy](./vars/createGithubIssue.groovy) | This library that identifies the failed components and creates the [AUTOCUT] issues. +| [publishGradleCheckTestResults.groovy](./vars/publishGradleCheckTestResults.groovy) | This library runs part of Gradle Check and publishes the failed test data to the [OpenSearch Metrics Cluster](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/e5e64d40-ed31-11ee-be99-69d1dbc75083). +| [gradleCheckFlakyTestDetector.groovy](./vars/gradleCheckFlakyTestDetector.groovy) | This library detects the flaky tests from [OpenSearch Metrics Cluster](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/e5e64d40-ed31-11ee-be99-69d1dbc75083) and generates a test report. +| [gradleCheckFlakyTestGitHubIssue.groovy](./vars/gradleCheckFlakyTestGitHubIssue.groovy) | This library is used in [gradleCheckFlakyTestDetector.groovy](./vars/gradleCheckFlakyTestDetector.groovy) to create/edit the GitHub Issue using the generated test report. ## Contributing diff --git a/build.gradle b/build.gradle index b5404de25..9d604a045 100644 --- a/build.gradle +++ b/build.gradle @@ -127,7 +127,7 @@ jacocoTestReport { } } -String version = '6.5.1' +String version = '6.5.2' task updateVersion { doLast { diff --git a/src/gradlecheck/FetchPostMergeFailedTestClass.groovy b/src/gradlecheck/FetchPostMergeFailedTestClass.groovy index f4d6660c6..faa4b3d42 100644 --- a/src/gradlecheck/FetchPostMergeFailedTestClass.groovy +++ b/src/gradlecheck/FetchPostMergeFailedTestClass.groovy @@ -27,7 +27,7 @@ class FetchPostMergeFailedTestClass { this.script = script } - def getQuery() { + def getQuery(timeFrame) { def queryMap = [ size: 200, query: [ @@ -52,11 +52,33 @@ class FetchPostMergeFailedTestClass { match: [ test_status: [ query: "FAILED", - operator: "OR" + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 ] ] ] - ] + ], + filter: [ + [ + range: [ + build_start_time: [ + from: "now-${timeFrame}", + to: "now", + include_lower: true, + include_upper: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 ] ], aggregations: [ @@ -68,13 +90,12 @@ class FetchPostMergeFailedTestClass { ] ] ] - def query = JsonOutput.toJson(queryMap) return query.replace('"', '\\"') } - def getPostMergeFailedTestClass() { - def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery()) + def getPostMergeFailedTestClass(timeFrame) { + def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery(timeFrame)) def keys = jsonResponse.aggregations.test_class_keyword_agg.buckets.collect { it.key } return keys } diff --git a/src/gradlecheck/MarkdownComparator.groovy b/src/gradlecheck/MarkdownComparator.groovy new file mode 100644 index 000000000..ecb2add8c --- /dev/null +++ b/src/gradlecheck/MarkdownComparator.groovy @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +class MarkdownComparator { + + ArrayList testReportMarkDownTable + ArrayList gitHubMarkDownTable + + MarkdownComparator(ArrayList testReportMarkDownTable, ArrayList gitHubMarkDownTable) { + this.testReportMarkDownTable = testReportMarkDownTable + this.gitHubMarkDownTable = gitHubMarkDownTable + } + + def markdownComparison() { + def differences = testReportMarkDownTable.findAll { ghRow -> + !gitHubMarkDownTable.any { compRow -> + ghRow['Git Reference'] == compRow['Git Reference'] && + ghRow['Merged Pull Request'] == compRow['Merged Pull Request'] && + ghRow['Build Details'] == compRow['Build Details'] && + ghRow['Test Name'] == compRow['Test Name'] + } + } + return differences + } +} diff --git a/src/gradlecheck/OpenSearchMetricsQuery.groovy b/src/gradlecheck/OpenSearchMetricsQuery.groovy index 72f6ac700..5038df653 100644 --- a/src/gradlecheck/OpenSearchMetricsQuery.groovy +++ b/src/gradlecheck/OpenSearchMetricsQuery.groovy @@ -26,14 +26,14 @@ class OpenSearchMetricsQuery { this.script = script } + // Ensure the alias `gradle-check` is created targeting all the gradle-check-* indices. def fetchMetrics(String query) { def response = script.sh( script: """ set -e set +x MONTH_YEAR=\$(date +"%m-%Y") - INDEX_NAME="gradle-check-\$MONTH_YEAR" - curl -s -XGET "${metricsUrl}/\$INDEX_NAME/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "${awsAccessKey}:${awsSecretKey}" -H "x-amz-security-token:${awsSessionToken}" -H 'Content-Type: application/json' -d "${query}" | jq '.' + curl -s -XGET "${metricsUrl}/gradle-check/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "${awsAccessKey}:${awsSecretKey}" -H "x-amz-security-token:${awsSessionToken}" -H 'Content-Type: application/json' -d "${query}" | jq '.' """, returnStdout: true ).trim() diff --git a/src/gradlecheck/ParseMarkDownTable.groovy b/src/gradlecheck/ParseMarkDownTable.groovy new file mode 100644 index 000000000..6912827e6 --- /dev/null +++ b/src/gradlecheck/ParseMarkDownTable.groovy @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +class ParseMarkDownTable { + String markdown + + ParseMarkDownTable(String markdown) { + this.markdown = markdown + } + + def parseMarkdownTableRows() { + def rows = markdown.split("\\|\\s*\\n") + rows = rows[2..-2] // Skipping headers and footer + return rows.collect { row -> + def cells = row.split("\\|\\s*") + return [ + 'Git Reference': cells[1].trim(), + 'Merged Pull Request': cells[2].trim(), + 'Build Details': cells[3].trim(), + 'Test Name': cells[4].trim() + ] + } + } +} \ No newline at end of file diff --git a/tests/gradlecheck/CreateMarkDownTableTest.groovy b/tests/gradlecheck/CreateMarkDownTableTest.groovy index c14c56df8..ae3afee17 100644 --- a/tests/gradlecheck/CreateMarkDownTableTest.groovy +++ b/tests/gradlecheck/CreateMarkDownTableTest.groovy @@ -22,11 +22,8 @@ class CreateMarkDownTableTest { [gitReference: "def456", pullRequestLink: "https://github.com/opensearch-project/OpenSearch/pull/2", buildDetailLink: "https://ci.opensearch.org/2", testNames: ["test3"]] ] def additionalPullRequests = ["3", "4"] - def createMarkDownTable = new CreateMarkDownTable(failedTest, tableData, additionalPullRequests) - def result = createMarkDownTable.createMarkdownTable() - def expectedOutput = """ ## Flaky Test Report for `ExampleTest` diff --git a/tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy b/tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy index aafbc2eb2..fe02dc8f4 100644 --- a/tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy +++ b/tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy @@ -71,11 +71,33 @@ class FetchPostMergeFailedTestClassTest { match: [ test_status: [ query: "FAILED", - operator: "OR" + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 ] ] ] - ] + ], + filter: [ + [ + range: [ + build_start_time: [ + from: "now-15d", + to: "now", + include_lower: true, + include_upper: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 ] ], aggregations: [ @@ -88,7 +110,7 @@ class FetchPostMergeFailedTestClassTest { ] ]).replace('"', '\\"') - def result = fetchPostMergeFailedTestClass.getQuery() + def result = fetchPostMergeFailedTestClass.getQuery("15d") assert result == expectedOutput } @@ -96,8 +118,8 @@ class FetchPostMergeFailedTestClassTest { @Test void testGetPostMergeFailedTestClassReturnsKeys() { def expectedOutput = ["testClass1", "testClass2"] - - def result = fetchPostMergeFailedTestClass.getPostMergeFailedTestClass() + def timeFrame = "15d" + def result = fetchPostMergeFailedTestClass.getPostMergeFailedTestClass(timeFrame) assert result == expectedOutput } diff --git a/tests/gradlecheck/MarkdownComparatorTest.groovy b/tests/gradlecheck/MarkdownComparatorTest.groovy new file mode 100644 index 000000000..16f36e0b9 --- /dev/null +++ b/tests/gradlecheck/MarkdownComparatorTest.groovy @@ -0,0 +1,208 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class MarkdownComparatorTest { + + @Test + public void testMarkdownComparisonWithDifferences() { + List> testReportMarkdown = new ArrayList<>(); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + testReportMarkdown.add(createRow("def456", "PR2", "Build2", "Test2")); + List> gitHubMarkdown = new ArrayList<>(); + gitHubMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(1, differences.size()); + Assert.assertEquals("def456", differences.get(0).get("Git Reference")); + Assert.assertEquals("PR2", differences.get(0).get("Merged Pull Request")); + Assert.assertEquals("Build2", differences.get(0).get("Build Details")); + Assert.assertEquals("Test2", differences.get(0).get("Test Name")); + } + + @Test + public void testMarkdownComparisonWithoutDifferences() { + List> testReportMarkdown = new ArrayList<>(); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + List> gitHubMarkdown = new ArrayList<>(); + gitHubMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(0, differences.size()); + } + + private Map createRow(String gitReference, String mergedPR, String buildDetails, String testName) { + Map row = new HashMap<>(); + row.put("Git Reference", gitReference); + row.put("Merged Pull Request", mergedPR); + row.put("Build Details", buildDetails); + row.put("Test Name", testName); + return row; + } + + @Test + public void testMarkdownComparisonWithPartialMatch() { + List> testReportMarkdown = new ArrayList<>(); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + testReportMarkdown.add(createRow("def456", "PR2", "Build2", "Test2")); + List> gitHubMarkdown = new ArrayList<>(); + gitHubMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + gitHubMarkdown.add(createRow("ghi789", "PR3", "Build3", "Test3")); + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(1, differences.size()); + Assert.assertEquals("def456", differences.get(0).get("Git Reference")); + Assert.assertEquals("PR2", differences.get(0).get("Merged Pull Request")); + Assert.assertEquals("Build2", differences.get(0).get("Build Details")); + Assert.assertEquals("Test2", differences.get(0).get("Test Name")); + } + + @Test + public void testMarkdownComparisonWithEmptyList() { + List> testReportMarkdown = new ArrayList<>(); + List> gitHubMarkdown = new ArrayList<>(); + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(0, differences.size()); + } + + @Test + public void testMarkdownComparisonWithDifferentOrder() { + List> testReportMarkdown = new ArrayList<>(); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + testReportMarkdown.add(createRow("def456", "PR2", "Build2", "Test2")); + List> gitHubMarkdown = new ArrayList<>(); + gitHubMarkdown.add(createRow("def456", "PR2", "Build2", "Test2")); + gitHubMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(0, differences.size()); + } + + @Test + public void testMarkdownComparisonWithIdenticalEntries() { + List> testReportMarkdown = new ArrayList<>(); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + List> gitHubMarkdown = new ArrayList<>(); + gitHubMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(0, differences.size()); + } + + @Test + public void testMarkdownComparisonWithNullValues() { + List> testReportMarkdown = new ArrayList<>(); + testReportMarkdown.add(createRow(null, "PR1", "Build1", "Test1")); + List> gitHubMarkdown = new ArrayList<>(); + gitHubMarkdown.add(createRow(null, "PR1", "Build1", "Test1")); + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(0, differences.size()); + } + + @Test + public void testMarkdownComparisonWithDifferentColumns() { + List> testReportMarkdown = new ArrayList<>(); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + List> gitHubMarkdown = new ArrayList<>(); + Map gitHubRow = createRow("abc123", "PR1", "Build1", "Test1"); + gitHubRow.put("Extra Column", "ExtraValue"); + gitHubMarkdown.add(gitHubRow); + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(0, differences.size()); + } + + @Test + public void testMarkdownComparisonCompleteMatch() { + List> testReportMarkdown = new ArrayList<>(); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + List> gitHubMarkdown = new ArrayList<>(); + gitHubMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(0, differences.size()); + } + + @Test + public void testMarkdownComparisonPartialMatch() { + List> testReportMarkdown = new ArrayList<>(); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + List> gitHubMarkdown = new ArrayList<>(); + gitHubMarkdown.add(createRow("abc123", "PR1", "Build1", "Test2")); // Different 'Test Name' + + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(1, differences.size()); + Assert.assertEquals("abc123", differences.get(0).get("Git Reference")); + } + + @Test + public void testMarkdownComparisonNoMatch() { + List> testReportMarkdown = new ArrayList<>(); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + List> gitHubMarkdown = new ArrayList<>(); + gitHubMarkdown.add(createRow("def456", "PR2", "Build2", "Test2")); + + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(1, differences.size()); + Assert.assertEquals("abc123", differences.get(0).get("Git Reference")); + } + + @Test + public void testMarkdownComparisonWithEmptyLists() { + List> testReportMarkdown = new ArrayList<>(); + List> gitHubMarkdown = new ArrayList<>(); + + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(0, differences.size()); + } + + @Test + public void testMarkdownComparisonMultipleMatches() { + List> testReportMarkdown = new ArrayList<>(); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); // Duplicate entry + testReportMarkdown.add(createRow("def456", "PR2", "Build2", "Test2")); + List> gitHubMarkdown = new ArrayList<>(); + gitHubMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); // Matching row + gitHubMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); // Another matching row + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(1, differences.size()); + Assert.assertEquals("def456", differences.get(0).get("Git Reference")); // Ensure correct difference + } + + @Test + public void testMarkdownComparisonDifferentDataTypes() { + List> testReportMarkdown = new ArrayList<>(); + testReportMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + testReportMarkdown.add(createRow(null, "PR2", "Build2", "Test2")); // Null value for Git Reference + List> gitHubMarkdown = new ArrayList<>(); + gitHubMarkdown.add(createRow("abc123", "PR1", "Build1", "Test1")); + gitHubMarkdown.add(createRow(null, "PR2", "Build2", "Test2")); + MarkdownComparator comparator = new MarkdownComparator(testReportMarkdown, gitHubMarkdown); + List> differences = comparator.markdownComparison(); + Assert.assertEquals(0, differences.size()); + } +} diff --git a/tests/gradlecheck/ParseMarkDownTableTest.groovy b/tests/gradlecheck/ParseMarkDownTableTest.groovy new file mode 100644 index 000000000..04281708f --- /dev/null +++ b/tests/gradlecheck/ParseMarkDownTableTest.groovy @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +import org.junit.Test +import static org.junit.Assert.assertEquals + +class ParseMarkDownTableTest { + + @Test + void testParseMarkdownTableRows() { + def markdown = """ + ## Flaky Test Report for `PitMultiNodeIT` + + Noticed the `PitMultiNodeIT` has some flaky, failing tests that failed during **post-merge actions**. + + ### Details + + | Git Reference | Merged Pull Request | Build Details | Test Name | + |---------------|----------------------|---------------|-----------| + | 708c37120bea33f258d1656132b9b05642c92720 | [14502](https://github.com/opensearch-project/OpenSearch/pull/14502) | [41571](https://build.ci.opensearch.org/job/gradle-check/41571/testReport/) | `org.opensearch.search.pit.PitMultiNodeIT.testCreatePitWhileNodeDropWithAllowPartialCreationFalse {p0={"search.concurrent_segment_search.enabled":"false"}}` | + + The other pull requests, besides those involved in post-merge actions, that contain failing tests with the `PitMultiNodeIT` class are: + + - [13801](https://github.com/opensearch-project/OpenSearch/pull/13801) + - [14362](https://github.com/opensearch-project/OpenSearch/pull/14362) + + For more details on the failed tests refer to [OpenSearch Gradle Check Metrics](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/e5e64d40-ed31-11ee-be99-69d1dbc75083) dashboard. + """ + ParseMarkDownTable parser = new ParseMarkDownTable(markdown) + def result = parser.parseMarkdownTableRows() + assertEquals("Expected 1 row in the result", 1, result.size()) + assertEquals("708c37120bea33f258d1656132b9b05642c92720", result[0]['Git Reference']) + assertEquals("[14502](https://github.com/opensearch-project/OpenSearch/pull/14502)", result[0]['Merged Pull Request']) + assertEquals("[41571](https://build.ci.opensearch.org/job/gradle-check/41571/testReport/)", result[0]['Build Details']) + assertEquals("`org.opensearch.search.pit.PitMultiNodeIT.testCreatePitWhileNodeDropWithAllowPartialCreationFalse {p0={\"search.concurrent_segment_search.enabled\":\"false\"}}`", result[0]['Test Name']) + } +} diff --git a/tests/jenkins/TestGradleCheckFlakyTestGitHubIssue.groovy b/tests/jenkins/TestGradleCheckFlakyTestGitHubIssue.groovy new file mode 100644 index 000000000..d65e1b971 --- /dev/null +++ b/tests/jenkins/TestGradleCheckFlakyTestGitHubIssue.groovy @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package jenkins.tests + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.MatcherAssert.assertThat +import com.lesfurets.jenkins.unit.* + +class TestGradleCheckFlakyTestGitHubIssue extends BuildPipelineTest { + + @Before + void setUp() { + super.setUp() + binding.setVariable('GITHUB_USER', 'dummy_user') + binding.setVariable('GITHUB_TOKEN', 'dummy_token') + } + + @Test + public void testDefaultIssueEdit() { + super.testPipeline("tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile", "tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueEdit_Jenkinsfile") + assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue edit bbb\nccc --repo https://github.com/prudhvigodithi/OpenSearch --body-file \"SampleTest.md\", returnStdout=true}")) + } + + @Test + public void testIssueCreate() { + helper.addShMock("""gh issue list --repo https://github.com/prudhvigodithi/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title" --json number --jq '.[0].number'""") { script -> + return [stdout: "", exitValue: 0] + } + helper.addShMock("""gh issue list --repo https://github.com/prudhvigodithi/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title is:closed" --json number --jq '.[0].number'""") { script -> + return [stdout: "", exitValue: 0] + } + super.testPipeline("tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile", "tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueCreate_Jenkinsfile") + assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue create --title \"[AUTOCUT] Gradle Check Flaky Test Report for SampleTest\" --body-file \"SampleTest.md\" --label \"autocut,>test-failure,flaky-test\" --label \"untriaged\" --repo https://github.com/prudhvigodithi/OpenSearch, returnStdout=true}")) + } + + def getCommands(method, text) { + def shCommands = helper.callStack.findAll { call -> + call.methodName == method + }.collect { call -> + callArgsToString(call) + }.findAll { command -> + command.contains(text) + } + return shCommands + } +} + diff --git a/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueCreate_Jenkinsfile.txt b/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueCreate_Jenkinsfile.txt new file mode 100644 index 000000000..f7190a2d6 --- /dev/null +++ b/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueCreate_Jenkinsfile.txt @@ -0,0 +1,12 @@ + TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.run() + TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.pipeline(groovy.lang.Closure) + TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.echo(Executing on agent [label:none]) + TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.stage(Detect Gradle Check Flaky Tests, groovy.lang.Closure) + TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.script(groovy.lang.Closure) + TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.gradleCheckFlakyTestGitHubIssue({repoUrl=https://github.com/prudhvigodithi/OpenSearch, issueTitle=[AUTOCUT] Gradle Check Flaky Test Report for SampleTest, issueBodyFile=SampleTest.md, label=autocut,>test-failure,flaky-test, issueEdit=true}) + gradleCheckFlakyTestGitHubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + gradleCheckFlakyTestGitHubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + gradleCheckFlakyTestGitHubIssue.sh({script=gh issue list --repo https://github.com/prudhvigodithi/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title" --json number --jq '.[0].number', returnStdout=true}) + gradleCheckFlakyTestGitHubIssue.sh({script=gh issue list --repo https://github.com/prudhvigodithi/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title is:closed" --json number --jq '.[0].number', returnStdout=true}) + gradleCheckFlakyTestGitHubIssue.println(Creating new issue) + gradleCheckFlakyTestGitHubIssue.sh({script=gh issue create --title "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest" --body-file "SampleTest.md" --label "autocut,>test-failure,flaky-test" --label "untriaged" --repo https://github.com/prudhvigodithi/OpenSearch, returnStdout=true}) diff --git a/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueEdit_Jenkinsfile.txt b/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueEdit_Jenkinsfile.txt new file mode 100644 index 000000000..359991c93 --- /dev/null +++ b/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueEdit_Jenkinsfile.txt @@ -0,0 +1,13 @@ + TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.run() + TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.pipeline(groovy.lang.Closure) + TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.echo(Executing on agent [label:none]) + TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.stage(Detect Gradle Check Flaky Tests, groovy.lang.Closure) + TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.script(groovy.lang.Closure) + TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.gradleCheckFlakyTestGitHubIssue({repoUrl=https://github.com/prudhvigodithi/OpenSearch, issueTitle=[AUTOCUT] Gradle Check Flaky Test Report for SampleTest, issueBodyFile=SampleTest.md, label=autocut,>test-failure,flaky-test, issueEdit=true}) + gradleCheckFlakyTestGitHubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + gradleCheckFlakyTestGitHubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + gradleCheckFlakyTestGitHubIssue.sh({script=gh issue list --repo https://github.com/prudhvigodithi/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title" --json number --jq '.[0].number', returnStdout=true}) + gradleCheckFlakyTestGitHubIssue.sh({script=gh issue list --repo https://github.com/prudhvigodithi/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title is:closed" --json number --jq '.[0].number', returnStdout=true}) + gradleCheckFlakyTestGitHubIssue.println(Issue already exists, editing the issue body) + gradleCheckFlakyTestGitHubIssue.sh({script=gh issue edit bbb +ccc --repo https://github.com/prudhvigodithi/OpenSearch --body-file "SampleTest.md", returnStdout=true}) diff --git a/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile b/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile new file mode 100644 index 000000000..282882966 --- /dev/null +++ b/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('Detect Gradle Check Flaky Tests') { + steps { + script { + gradleCheckFlakyTestGitHubIssue( + repoUrl: "https://github.com/prudhvigodithi/OpenSearch", + issueTitle: "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest", + issueBodyFile: "SampleTest.md", + label: 'autocut,>test-failure,flaky-test', + issueEdit: true + ) + } + } + } + } +} diff --git a/vars/gradleCheckFlakyTestDetector.groovy b/vars/gradleCheckFlakyTestDetector.groovy index 0b7cd4e0a..563141ea6 100644 --- a/vars/gradleCheckFlakyTestDetector.groovy +++ b/vars/gradleCheckFlakyTestDetector.groovy @@ -10,6 +10,7 @@ /** Library to detect Gradle Check flaky tests and create GitHub issue in OpenSearch repository. @param Map args = [:] args A map of the following parameters @param args.issueLabels - GitHub labels that will be added to the issue created in OpenSearch repository. + @param args.timeFrame - The time frame for the query range, specified in OpenSearch date math syntax (e.g., "15d" for 15 days). */ import gradlecheck.FetchPostMergeFailedTestClass @@ -28,7 +29,8 @@ void call(Map args = [:]) { def awsAccessKey = env.AWS_ACCESS_KEY_ID def awsSecretKey = env.AWS_SECRET_ACCESS_KEY def awsSessionToken = env.AWS_SESSION_TOKEN - def postMergeFailedTests = new FetchPostMergeFailedTestClass(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, this).getPostMergeFailedTestClass() + def timeFrame = args.timeFrame ?: '30d' + def postMergeFailedTests = new FetchPostMergeFailedTestClass(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, this).getPostMergeFailedTestClass(timeFrame) postMergeFailedTests.each { failedTest -> def testData = [] def allPullRequests = [] @@ -50,7 +52,7 @@ void call(Map args = [:]) { def testNameAdditionalPullRequests = new FetchTestPullRequests(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, this).getTestPullRequests(failedTest).findAll { !allPullRequests.contains(it) } def markdownTable = new CreateMarkDownTable(failedTest, testData, testNameAdditionalPullRequests).createMarkdownTable() writeFile file: "${failedTest}.md", text: markdownTable - createGithubIssue( + gradleCheckFlakyTestGitHubIssue( repoUrl: "https://github.com/opensearch-project/OpenSearch", issueTitle: "[AUTOCUT] Gradle Check Flaky Test Report for ${failedTest}", issueBodyFile: "${failedTest}.md", diff --git a/vars/gradleCheckFlakyTestGitHubIssue.groovy b/vars/gradleCheckFlakyTestGitHubIssue.groovy new file mode 100644 index 000000000..f9fded652 --- /dev/null +++ b/vars/gradleCheckFlakyTestGitHubIssue.groovy @@ -0,0 +1,88 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Library to create/edit/skip Flaky Test Report GitHub issue for OpenSearch repo. + @param Map args = [:] args A map of the following parameters + @param args.repoUrl - GitHub repository URL to create issue + @param args.issueTitle - GitHub issue title + @param args.issueBody - GitHub issue body + @param args.label - GitHub issue label to be attached along with 'untriaged'. Defaults to autocut. + @param args.issueEdit - Updates the body of the issue, the default if not passed is to add a comment. + @param args.issueBodyFile - GitHub issue body from an `.md` file + */ + +import gradlecheck.ParseMarkDownTable +import gradlecheck.MarkdownComparator + +void call(Map args = [:]) { + label = args.label ?: 'autocut,>test-failure,flaky-test' + try { + withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { + def openIssue = sh( + script: "gh issue list --repo ${args.repoUrl} -S \"${args.issueTitle} in:title\" --json number --jq '.[0].number'", + returnStdout: true + ).trim() + + + def closedIssue = sh( + script: "gh issue list --repo ${args.repoUrl} -S \"${args.issueTitle} in:title is:closed\" --json number --jq '.[0].number'", + returnStdout: true + ).trim() + + if (openIssue) { + println('Issue already exists, editing the issue body') + sh( + script: "gh issue edit ${openIssue} --repo ${args.repoUrl} --body-file \"${args.issueBodyFile}\"", + returnStdout: true + ) + } + else if (!openIssue && closedIssue) { + def existingIssueBody = sh( + script: "gh issue list --repo ${args.repoUrl} -S \"${args.issueTitle} in:title is:closed\" --json body --jq '.[0].body'", + returnStdout: true + ).trim() + def existingTable = new ParseMarkDownTable(existingIssueBody).parseMarkdownTableRows() + def markdownTable = new ParseMarkDownTable(readFile(args.issueBodyFile)).parseMarkdownTableRows() + def differences = new MarkdownComparator(markdownTable, existingTable).markdownComparison() + if (!differences) { + println("Not Re-opening the issue as the no change in the Flaky report after the issue is closed") + } else { + println "Differences found:" + differences.each { diffRow -> + println "Git Reference: ${diffRow['Git Reference']}, " + + "Merged Pull Request: ${diffRow['Merged Pull Request']}, " + + "Build Details: ${diffRow['Build Details']}, " + + "Test Name: ${diffRow['Test Name']}" + } + sh( + script: "gh issue reopen --repo ${args.repoUrl} ${closedIssue}", + returnStdout: true + ) + sh( + script: "gh issue edit ${closedIssue} --repo ${args.repoUrl} --body-file \"${args.issueBodyFile}\"", + returnStdout: true + ) + } + } + else { + println("Creating new issue") + sh( + script: "gh issue create --title \"${args.issueTitle}\" --body-file \"${args.issueBodyFile}\" --label \"${label}\" --label \"untriaged\" --repo ${args.repoUrl}", + returnStdout: true + ) + } + } + } catch (Exception ex) { + error("Unable to create GitHub issue for ${args.repoUrl}", ex.getMessage()) + } +} + + + +