Merge branch 'main' into 4967_prevent_deletion_of_snapshot_backing_index

.ci/bwcVersions

@@ -53,4 +53,5 @@ BWC_VERSION:
   - "2.3.0"
   - "2.3.1"
   - "2.4.0"
+  - "2.4.1"
   - "2.5.0"

.github/workflows/gradle-check.yml

@@ -67,7 +67,7 @@ jobs:
           files: ./codeCoverage.xml
 
       - name: Create Comment Success
-        if: ${{ github.event_name == 'pull_request_target' && success() }}
+        if: ${{ github.event_name == 'pull_request_target' && success() && env.result == 'SUCCESS' }}
         uses: peter-evans/create-or-update-comment@v2
         with:
           issue-number: ${{ env.pr_number }}
@@ -77,15 +77,42 @@ jobs:
               * **URL:** ${{ env.workflow_url }}
               * **CommitID:** ${{ env.pr_from_sha }}
 
+      - name: Extract Test Failure
+        if: ${{ github.event_name == 'pull_request_target' && env.result != 'SUCCESS' }}
+        run: |
+            TEST_FAILURES=`curl -s "${{ env.workflow_url }}/testReport/api/json?tree=suites\[cases\[status,className,name\]\]" | jq -r '.. | objects | select(.status=="FAILED",.status=="REGRESSION") | (.className + "." +  .name)' | uniq -c | sort -n -r | head -n 10`
+            if [[ "$TEST_FAILURES" != "" ]]
+            then
+              echo "test_failures<<EOF" >> $GITHUB_ENV
+              echo "" >> $GITHUB_ENV
+              echo "* **TEST FAILURES:**" >> $GITHUB_ENV
+              echo '```' >> $GITHUB_ENV
+              echo "$TEST_FAILURES" >> $GITHUB_ENV
+              echo '```' >> $GITHUB_ENV
+              echo "EOF" >> $GITHUB_ENV
+            fi
+
+      - name: Create Comment Flaky
+        if: ${{ github.event_name == 'pull_request_target' && success() && env.result != 'SUCCESS' }}
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ env.pr_number }}
+          body: |
+            ### Gradle Check (Jenkins) Run Completed with:
+            * **RESULT:** ${{ env.result }} :grey_exclamation: ${{ env.test_failures }}
+            * **URL:** ${{ env.workflow_url }}
+            * **CommitID:** ${{ env.pr_from_sha }}
+            Please review all [flaky tests](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) that succeeded after retry and create an issue if one does not already exist to track the flaky failure.
+
       - name: Create Comment Failure
         if: ${{ github.event_name == 'pull_request_target' && failure() }}
         uses: peter-evans/create-or-update-comment@v2
         with:
           issue-number: ${{ env.pr_number }}
           body: |
               ### Gradle Check (Jenkins) Run Completed with:
-              * **RESULT:** ${{ env.result }} :x:
+              * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }}
               * **URL:** ${{ env.workflow_url }}
               * **CommitID:** ${{ env.pr_from_sha }}
-              Please examine the workflow log, locate, and copy-paste the failure below, then iterate to green.
+              Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
               Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?

CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## [Unreleased]
+### Dependencies
+- Bumps `commons-compress` from 1.21 to 1.22
+- Bumps `jcodings` from 1.0.57 to 1.0.58
+- Bumps `google-http-client-jackson2` from 1.35.0 to 1.42.3
+
 All notable changes to this project are documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See the [CONTRIBUTING guide](./CONTRIBUTING.md#Changelog) for instructions on how to add changelog entries.
@@ -21,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Prevent deletion of snapshots that are backing searchable snapshot indexes ([#5069](https://github.com/opensearch-project/OpenSearch/pull/5069))
 - Integration tests for searchable snapshots ([#5048](https://github.com/opensearch-project/OpenSearch/pull/5048))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
+- Add feature flag for extensions ([#5211](https://github.com/opensearch-project/OpenSearch/pull/5211))
 
 >>>>>>> main
 ### Dependencies
@@ -84,11 +91,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bumps `commons-compress` from 1.21 to 1.22 ([#5104](https://github.com/opensearch-project/OpenSearch/pull/5104))
 - Bump `opencensus-contrib-http-util` from 0.18.0 to 0.31.1 ([#3633](https://github.com/opensearch-project/OpenSearch/pull/3633))
 - Bump `geoip2` from 3.0.1 to 3.0.2 ([#5103](https://github.com/opensearch-project/OpenSearch/pull/5103))
+- Bump gradle-extra-configurations-plugin from 7.0.0 to 8.0.0 ([#4808](https://github.com/opensearch-project/OpenSearch/pull/4808))
 ### Changed
 ### Deprecated
 ### Removed
 ### Fixed
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.4...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.4...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.4...2.x

buildSrc/build.gradle

@@ -105,7 +105,7 @@ dependencies {
   api 'commons-codec:commons-codec:1.15'
   api 'org.apache.commons:commons-compress:1.21'
   api 'org.apache.ant:ant:1.10.12'
-  api 'com.netflix.nebula:gradle-extra-configurations-plugin:7.0.0'
+  api 'com.netflix.nebula:gradle-extra-configurations-plugin:8.0.0'
   api 'com.netflix.nebula:nebula-publishing-plugin:4.6.0'
   api 'com.netflix.nebula:gradle-info-plugin:11.3.3'
   api 'org.apache.rat:apache-rat:0.13'

buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java

@@ -51,8 +51,7 @@
 
 import java.util.Arrays;
 import java.util.Comparator;
-import java.util.List;
-import java.util.stream.Collectors;
+import java.util.Objects;
 
 /**
  * A plugin to manage getting and extracting distributions of OpenSearch.
@@ -71,12 +70,6 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
     private static final String SNAPSHOT_REPO_NAME = "opensearch-snapshots";
     public static final String DISTRO_EXTRACTED_CONFIG_PREFIX = "opensearch_distro_extracted_";
 
-    // for downloading Elasticsearch OSS distributions to run BWC
-    private static final String FAKE_IVY_GROUP_ES = "elasticsearch-distribution";
-    private static final String DOWNLOAD_REPO_NAME_ES = "elasticsearch-downloads";
-    private static final String SNAPSHOT_REPO_NAME_ES = "elasticsearch-snapshots";
-    private static final String FAKE_SNAPSHOT_IVY_GROUP_ES = "elasticsearch-distribution-snapshot";
-
     private static final String RELEASE_PATTERN_LAYOUT = "/core/opensearch/[revision]/[module]-min-[revision](-[classifier]).[ext]";
     private static final String SNAPSHOT_PATTERN_LAYOUT =
         "/snapshots/core/opensearch/[revision]/[module]-min-[revision](-[classifier])-latest.[ext]";
@@ -159,35 +152,20 @@ private DistributionDependency resolveDependencyNotation(Project p, OpenSearchDi
         return distributionsResolutionStrategiesContainer.stream()
             .sorted(Comparator.comparingInt(DistributionResolution::getPriority))
             .map(r -> r.getResolver().resolve(p, distribution))
-            .filter(d -> d != null)
+            .filter(Objects::nonNull)
             .findFirst()
             .orElseGet(() -> DistributionDependency.of(dependencyNotation(distribution)));
     }
 
     private static void addIvyRepo(Project project, String name, String url, String group, String... patternLayout) {
-        final List<IvyArtifactRepository> repos = Arrays.stream(patternLayout).map(pattern -> project.getRepositories().ivy(repo -> {
-            repo.setName(name);
-            repo.setUrl(url);
-            repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
-            repo.patternLayout(layout -> layout.artifact(pattern));
-        })).collect(Collectors.toList());
-
         project.getRepositories().exclusiveContent(exclusiveContentRepository -> {
             exclusiveContentRepository.filter(config -> config.includeGroup(group));
-            exclusiveContentRepository.forRepositories(repos.toArray(new IvyArtifactRepository[repos.size()]));
-        });
-    }
-
-    private static void addIvyRepo2(Project project, String name, String url, String group) {
-        IvyArtifactRepository ivyRepo = project.getRepositories().ivy(repo -> {
-            repo.setName(name);
-            repo.setUrl(url);
-            repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
-            repo.patternLayout(layout -> layout.artifact("/downloads/elasticsearch/elasticsearch-oss-[revision](-[classifier]).[ext]"));
-        });
-        project.getRepositories().exclusiveContent(exclusiveContentRepository -> {
-            exclusiveContentRepository.filter(config -> config.includeGroup(group));
-            exclusiveContentRepository.forRepositories(ivyRepo);
+            exclusiveContentRepository.forRepositories(Arrays.stream(patternLayout).map(pattern -> project.getRepositories().ivy(repo -> {
+                repo.setName(name);
+                repo.setUrl(url);
+                repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+                repo.patternLayout(layout -> layout.artifact(pattern));
+            })).toArray(IvyArtifactRepository[]::new));
         });
     }
 
@@ -211,9 +189,6 @@ private static void setupDownloadServiceRepo(Project project) {
             );
             addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://artifacts.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, SNAPSHOT_PATTERN_LAYOUT);
         }
-
-        addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES);
-        addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES);
     }
 
     /**
@@ -222,59 +197,37 @@ private static void setupDownloadServiceRepo(Project project) {
      * The returned object is suitable to be passed to {@link DependencyHandler}.
      * The concrete type of the object will be a set of maven coordinates as a {@link String}.
      * Maven coordinates point to either the integ-test-zip coordinates on maven central, or a set of artificial
-     * coordinates that resolve to the Elastic download service through an ivy repository.
+     * coordinates that resolve to the OpenSearch download service through an ivy repository.
      */
     private String dependencyNotation(OpenSearchDistribution distribution) {
         Version distroVersion = Version.fromString(distribution.getVersion());
         if (distribution.getType() == Type.INTEG_TEST_ZIP) {
-            if (distroVersion.onOrAfter("1.0.0")) {
-                return "org.opensearch.distribution.integ-test-zip:opensearch:" + distribution.getVersion() + "@zip";
-            } else {
-                return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip";
-            }
+            return "org.opensearch.distribution.integ-test-zip:opensearch:" + distribution.getVersion() + "@zip";
         }
 
         String extension = distribution.getType().toString();
         String classifier = distroVersion.onOrAfter("1.0.0") ? ":x64" : ":x86_64";
         if (distribution.getType() == Type.ARCHIVE) {
             extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
 
-            if (distroVersion.onOrAfter("1.0.0")) {
-                switch (distribution.getArchitecture()) {
-                    case ARM64:
-                        classifier = ":" + distribution.getPlatform() + "-arm64";
-                        break;
-                    case X64:
-                        classifier = ":" + distribution.getPlatform() + "-x64";
-                        break;
-                    case S390X:
-                        classifier = ":" + distribution.getPlatform() + "-s390x";
-                        break;
-                    default:
-                        throw new IllegalArgumentException("Unsupported architecture: " + distribution.getArchitecture());
-                }
-            } else if (distroVersion.onOrAfter("7.0.0")) {
-                classifier = ":" + distribution.getPlatform() + "-x86_64";
-            } else {
-                classifier = "";
+            switch (distribution.getArchitecture()) {
+                case ARM64:
+                    classifier = ":" + distribution.getPlatform() + "-arm64";
+                    break;
+                case X64:
+                    classifier = ":" + distribution.getPlatform() + "-x64";
+                    break;
+                case S390X:
+                    classifier = ":" + distribution.getPlatform() + "-s390x";
+                    break;
+                default:
+                    throw new IllegalArgumentException("Unsupported architecture: " + distribution.getArchitecture());
             }
         } else if (distribution.getType() == Type.DEB) {
-            if (distroVersion.onOrAfter("7.0.0")) {
-                classifier = ":amd64";
-            } else {
-                classifier = "";
-            }
-        } else if (distribution.getType() == Type.RPM && distroVersion.before("7.0.0")) {
-            classifier = "";
+            classifier = ":amd64";
         }
 
-        String group;
-        if (distroVersion.onOrAfter("1.0.0")) {
-            group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
-            return group + ":opensearch" + ":" + distribution.getVersion() + classifier + "@" + extension;
-        } else {
-            group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP_ES : FAKE_IVY_GROUP_ES;
-            return group + ":elasticsearch-oss" + ":" + distribution.getVersion() + classifier + "@" + extension;
-        }
+        String group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
+        return group + ":opensearch" + ":" + distribution.getVersion() + classifier + "@" + extension;
     }
 }

buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

@@ -109,7 +109,9 @@ public void execute(Task t) {
                         test.systemProperty("java.locale.providers", "SPI,JRE");
                     } else {
                         test.systemProperty("java.locale.providers", "SPI,COMPAT");
-                        test.jvmArgs("--illegal-access=warn");
+                        if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) < 0) {
+                            test.jvmArgs("--illegal-access=warn");
+                        }
                     }
                     if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) > 0) {
                         test.jvmArgs("-Djava.security.manager=allow");

buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java

@@ -32,7 +32,6 @@
 package org.opensearch.gradle.testclusters;
 
 import org.opensearch.gradle.FileSupplier;
-import org.opensearch.gradle.Jdk;
 import org.opensearch.gradle.PropertyNormalization;
 import org.opensearch.gradle.ReaperService;
 import org.opensearch.gradle.http.WaitForHttpResource;
@@ -75,7 +74,6 @@ public class OpenSearchCluster implements TestClusterConfiguration, Named {
     private final String path;
     private final String clusterName;
     private final NamedDomainObjectContainer<OpenSearchNode> nodes;
-    private final Jdk bwcJdk;
     private final File workingDirBase;
     private final LinkedHashMap<String, Predicate<TestClusterConfiguration>> waitConditions = new LinkedHashMap<>();
     private final Project project;
@@ -92,8 +90,7 @@ public OpenSearchCluster(
         ReaperService reaper,
         File workingDirBase,
         FileSystemOperations fileSystemOperations,
-        ArchiveOperations archiveOperations,
-        Jdk bwcJdk
+        ArchiveOperations archiveOperations
     ) {
         this.path = project.getPath();
         this.clusterName = clusterName;
@@ -103,7 +100,6 @@ public OpenSearchCluster(
         this.archiveOperations = archiveOperations;
         this.workingDirBase = workingDirBase;
         this.nodes = project.container(OpenSearchNode.class);
-        this.bwcJdk = bwcJdk;
 
         // Always add the first node
         String zone = hasZoneProperty() ? "zone-1" : "";
@@ -167,7 +163,6 @@ private void addNode(String nodeName, String zoneName) {
             fileSystemOperations,
             archiveOperations,
             workingDirBase,
-            bwcJdk,
             zoneName
         );
         // configure the cluster name eagerly