Merge branch 'main' into issue710

.github/workflows/docker-security-test-workflow.yml

@@ -66,9 +66,10 @@ jobs:
       - name: Run Index Management Test for security enabled test cases
         if: env.imagePresent == 'true'
         run: |
-          cluster_running=`curl -XGET https://localhost:9200/_cat/plugins -u admin:admin --insecure`
-          echo $cluster_running
-          security=`curl -XGET https://localhost:9200/_cat/plugins -u admin:admin --insecure |grep opensearch-security|wc -l`
+          container_id=`docker ps -q`
+          plugins=`docker exec $container_id /usr/share/opensearch/bin/opensearch-plugin list`
+          echo "plugins: $plugins"
+          security=`echo $plugins | grep opensearch-security | wc -l`
           echo $security
           if [ $security -gt 0 ]
           then

.github/workflows/multi-node-test-workflow.yml

@@ -9,7 +9,13 @@ on:
       - "**"
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
   multi-node-test:
+    needs: Get-CI-Image-Tag
     env:
       TEST_FILTER: ${{ matrix.test_filter }}
     strategy:
@@ -23,6 +29,13 @@ jobs:
             test_filter: -PexcludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
     # This job runs on Linux
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK
@@ -34,7 +47,9 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       - name: Run integration tests with multi node config
-        run: ./gradlew integTest -PnumNodes=3 ${{ env.TEST_FILTER }}
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew integTest -PnumNodes=3 ${{ env.TEST_FILTER }}"
       - name: Upload failed logs
         uses: actions/upload-artifact@v2
         if: failure()

.github/workflows/security-test-workflow.yml

@@ -9,9 +9,22 @@ on:
       - "**"
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
   security-test:
+    needs: Get-CI-Image-Tag
     # This job runs on Linux
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK
@@ -24,7 +37,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Run integration tests
         run: | 
-          ./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*SecurityBehaviorIT'
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*SecurityBehaviorIT'"
       - name: Upload failed logs
         uses: actions/upload-artifact@v2
         if: failure()

.github/workflows/test-and-build-workflow.yml

@@ -8,7 +8,73 @@ on:
       - "**"
 
 jobs:
-  test-and-build:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  test-and-build-linux:
+    needs: Get-CI-Image-Tag
+    env:
+      TEST_FILTER: ${{ matrix.test_filter }}
+    strategy:
+      # This setting says that all jobs should finish, even if one fails
+      fail-fast: false
+      # This starts three jobs, setting these environment variables uniquely for the different jobs
+      matrix:
+        java: [11, 17]
+        feature: [ism, non-ism]
+        include:
+          - feature: ism
+            test_filter: -PincludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
+          - feature: non-ism
+            test_filter: -PexcludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v2
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: ${{ matrix.java }}
+      # build index management
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      # This is a hack, but this step creates a link to the X: mounted drive, which makes the path
+      # short enough to work on Windows
+      - name: Build with Gradle
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew build ${{ env.TEST_FILTER }}"
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: logs
+          path: build/testclusters/integTest-*/logs/*
+      - name: Create Artifact Path
+        run: |
+          mkdir -p index-management-artifacts
+          cp ./build/distributions/*.zip index-management-artifacts
+      - name: Uploads coverage
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: index-management-plugin-ubuntu-latest
+          path: index-management-artifacts
+
+  test-and-build-windows-macos:
     env:
       BUILD_ARGS: ${{ matrix.os_build_args }}
       WORKING_DIR: ${{ matrix.working_directory }}.
@@ -19,7 +85,7 @@ jobs:
       # This starts three jobs, setting these environment variables uniquely for the different jobs
       matrix:
         java: [11, 17]
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [windows-latest, macos-latest]
         feature: [ism, non-ism]
         include:
           - os: windows-latest
@@ -53,21 +119,10 @@ jobs:
         run: ./gradlew build ${{ env.BUILD_ARGS }} ${{ env.TEST_FILTER }}
         env:
           _JAVA_OPTIONS: ${{ matrix.os_java_options }}
-      - name: Upload failed logs
-        uses: actions/upload-artifact@v2
-        if: ${{ failure() && matrix.os == 'ubuntu-latest' }}
-        with:
-          name: logs
-          path: build/testclusters/integTest-*/logs/*
       - name: Create Artifact Path
         run: |
           mkdir -p index-management-artifacts
           cp ./build/distributions/*.zip index-management-artifacts
-      - name: Uploads coverage
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        uses: codecov/codecov-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
       # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
       - name: Upload Artifacts
         uses: actions/upload-artifact@v1

DEVELOPER_GUIDE.md

@@ -48,7 +48,7 @@ However, to build the `index management` plugin project, we also use the OpenSea
 
 ### Building from the command line
 
-1. `./gradlew build` builds and tests project.
+1. `./gradlew build` builds and tests project. 
 2. `./gradlew run` launches a single node cluster with the index management (and job-scheduler) plugin installed.
 3. `./gradlew run -PnumNodes=3` launches a multi-node cluster with the index management (and job-scheduler) plugin installed.
 4. `./gradlew integTest` launches a single node cluster with the index management (and job-scheduler) plugin installed and runs all integ tests.

build.gradle

@@ -628,7 +628,7 @@ task integTestRemote(type: RestIntegTestTask) {
 
 // === Set up BWC tests ===
 
-String bwcVersionShort = "2.11.0"
+String bwcVersionShort = "2.12.0"
 String bwcVersion = bwcVersionShort + ".0"
 String baseName = "indexmanagementBwcCluster"
 

release-notes/opensearch-index-management.release-notes-2.11.0.0.md

@@ -0,0 +1,21 @@
+## Version 2.11.0.0 2023-10-11
+
+Compatible with OpenSearch 2.10.0
+
+### Maintenance
+* Increment version to 2.11.0-SNAPSHOT. ([#922](https://github.com/opensearch-project/index-management/pull/922))
+
+### Enhancements
+* Provide unique id for each rollup job and add debug logs. ([#968](https://github.com/opensearch-project/index-management/pull/968))
+
+### Bug fixes
+* Fix auto managed index always have -2 seqNo bug. ([#924](https://github.com/opensearch-project/index-management/pull/924))
+
+### Infrastructure
+* Upload docker test cluster log. ([#964](https://github.com/opensearch-project/index-management/pull/964))
+* Reduce test running time. ([#965](https://github.com/opensearch-project/index-management/pull/965))
+* Parallel test run. ([#966](https://github.com/opensearch-project/index-management/pull/966))
+* Security test filtered. ([#969](https://github.com/opensearch-project/index-management/pull/969))
+
+### Documentation
+* Added 2.11 release notes. ([#1004](https://github.com/opensearch-project/index-management/pull/1004))