Merge branch 'opensearch-project:main' into main-bulkIndexFindings

.github/workflows/security-test-workflow.yml

@@ -12,7 +12,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [ 11, 17 ]
+        java: [ 11, 17, 21 ]
     # Job name
     name: Build and test Alerting
     # This job runs on Linux

.github/workflows/test-workflow.yml

@@ -21,7 +21,7 @@ jobs:
       WORKING_DIR: ${{ matrix.working_directory }}.
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
     # Job name
     name: Build Alerting with JDK ${{ matrix.java }} on Linux
     # This job runs on Linux
@@ -69,7 +69,7 @@ jobs:
       WORKING_DIR: ${{ matrix.working_directory }}.
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
         os: [ windows-latest, macos-latest ]
         include:
           - os: windows-latest

alerting/build.gradle

@@ -16,7 +16,7 @@ apply plugin: 'jacoco'
 
 def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster')
 def usingMultiNode = project.properties.containsKey('numNodes')
-String bwcVersion = "2.10.0.0"
+String bwcVersion = "2.12.0.0"
 
 ext {
     projectSubstitutions = [:]
@@ -260,7 +260,7 @@ String baseName = "alertingBwcCluster"
     testClusters {
         "${baseName}$i" {
             testDistribution = "ARCHIVE"
-            versions = ["2.10.0-SNAPSHOT", "3.0.0-SNAPSHOT"]
+            versions = ["2.12.0-SNAPSHOT", "3.0.0-SNAPSHOT"]
             numberOfNodes = 3
             plugin(provider(new Callable<RegularFile>(){
                 @Override

alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt

@@ -119,7 +119,7 @@ class TransportDeleteWorkflowAction @Inject constructor(
     ) {
         suspend fun resolveUserAndStart() {
             try {
-                val workflow = getWorkflow()
+                val workflow: Workflow = getWorkflow() ?: return
 
                 val canDelete = user == null ||
                     !doFilterForUser(user) ||
@@ -296,17 +296,27 @@ class TransportDeleteWorkflowAction @Inject constructor(
             return deletableMonitors
         }
 
-        private suspend fun getWorkflow(): Workflow {
+        private suspend fun getWorkflow(): Workflow? {
             val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, workflowId)
 
             val getResponse: GetResponse = client.suspendUntil { get(getRequest, it) }
-            if (getResponse.isExists == false) {
-                actionListener.onFailure(
-                    AlertingException.wrap(
-                        OpenSearchStatusException("Workflow not found.", RestStatus.NOT_FOUND)
-                    )
-                )
+            if (!getResponse.isExists) {
+                handleWorkflowMissing()
+                return null
             }
+
+            return parseWorkflow(getResponse)
+        }
+
+        private fun handleWorkflowMissing() {
+            actionListener.onFailure(
+                AlertingException.wrap(
+                    OpenSearchStatusException("Workflow not found.", RestStatus.NOT_FOUND)
+                )
+            )
+        }
+
+        private fun parseWorkflow(getResponse: GetResponse): Workflow {
             val xcp = XContentHelper.createParser(
                 xContentRegistry, LoggingDeprecationHandler.INSTANCE,
                 getResponse.sourceAsBytesRef, XContentType.JSON

alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt

@@ -27,6 +27,7 @@ import org.opensearch.commons.alerting.model.Workflow
 import org.opensearch.commons.authuser.User
 import org.opensearch.commons.utils.recreateObject
 import org.opensearch.core.action.ActionListener
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry
 import org.opensearch.index.query.BoolQueryBuilder
 import org.opensearch.index.query.ExistsQueryBuilder
 import org.opensearch.index.query.MatchQueryBuilder
@@ -41,7 +42,8 @@ class TransportSearchMonitorAction @Inject constructor(
     val settings: Settings,
     val client: Client,
     clusterService: ClusterService,
-    actionFilters: ActionFilters
+    actionFilters: ActionFilters,
+    val namedWriteableRegistry: NamedWriteableRegistry
 ) : HandledTransportAction<ActionRequest, SearchResponse>(
     AlertingActions.SEARCH_MONITORS_ACTION_NAME, transportService, actionFilters, ::SearchMonitorRequest
 ),
@@ -54,7 +56,7 @@ class TransportSearchMonitorAction @Inject constructor(
 
     override fun doExecute(task: Task, request: ActionRequest, actionListener: ActionListener<SearchResponse>) {
         val transformedRequest = request as? SearchMonitorRequest
-            ?: recreateObject(request) {
+            ?: recreateObject(request, namedWriteableRegistry) {
                 SearchMonitorRequest(it)
             }
 

alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt

@@ -45,7 +45,6 @@ class DestinationMigrationCoordinator(
     }
 
     override fun clusterChanged(event: ClusterChangedEvent) {
-        logger.info("Detected cluster change event for destination migration")
         if (DestinationMigrationUtilService.finishFlag) {
             logger.info("Reset destination migration process.")
             scheduledMigration?.cancel()
@@ -63,6 +62,7 @@ class DestinationMigrationCoordinator(
                 runningLock = false
             }
         } else if (!event.localNodeClusterManager()) {
+            logger.info("Cancelling the migration process.")
             scheduledMigration?.cancel()
         }
     }

build-tools/merged-coverage.gradle

@@ -5,7 +5,7 @@
 
 allprojects {
     plugins.withId('jacoco') {
-        jacoco.toolVersion = '0.8.7'
+        jacoco.toolVersion = '0.8.11'
         // For some reason this dependency isn't getting setup automatically by the jacoco plugin
         tasks.withType(JacocoReport) { 
             dependsOn tasks.withType(Test)

build.gradle

@@ -39,7 +39,7 @@ buildscript {
 }
 
 plugins {
-    id 'com.netflix.nebula.ospackage' version "11.0.0"
+    id 'com.netflix.nebula.ospackage' version "11.5.0"
     id "com.dorongold.task-tree" version "1.5"
 }
 
@@ -48,7 +48,12 @@ apply plugin: 'jacoco'
 apply from: 'build-tools/merged-coverage.gradle'
 
 configurations {
-    ktlint
+    ktlint {
+      resolutionStrategy {
+          force "ch.qos.logback:logback-classic:1.3.14"
+          force "ch.qos.logback:logback-core:1.3.14"
+      }
+   }
 }
 
 dependencies {

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
 networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -83,7 +83,8 @@ done
 # This is normally unused
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -130,26 +131,29 @@ location of your Java installation."
     fi
 else
     JAVACMD=java
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+    fi
 fi
 
 # Increase the maximum file descriptors if we can.
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
         # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045
+        # shellcheck disable=SC2039,SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
         # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045
+        # shellcheck disable=SC2039,SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -198,11 +202,11 @@ fi
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
-# Collect all arguments for the java command;
-#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
-#     shell script including quotes and variable substitutions, so put them in
-#     double quotes to make sure that they get re-expanded; and
-#   * put everything else in single quotes, so that it's not re-expanded.
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \