Merge branch 'opensearch-project:main' into main

.ci/bwcVersions

@@ -37,3 +37,12 @@ BWC_VERSION:
   - "1.2.4"
   - "1.2.5"
   - "1.3.0"
+  - "1.3.1"
+  - "1.3.2"
+  - "1.3.3"
+  - "1.3.4"
+  - "2.0.0"
+  - "2.0.1"
+  - "2.0.2"
+  - "2.1.0"
+  - "2.2.0"

.ci/dockerOnLinuxExclusions

@@ -12,4 +12,4 @@ ol-7.7
 sles-12.3 # older version used in Vagrant image
 sles-12.5
 sles-15.1
-sles-15.2
+sles-15.2

.ci/documentation/issue.md

@@ -0,0 +1,11 @@
+**Is your feature request related to a problem?**
+A new feature has been added.
+
+**What solution would you like?**
+Document the usage of the new feature.
+
+**What alternatives have you considered?**
+N/A
+
+**Do you have any additional context?**
+See please 

.github/CODEOWNERS

@@ -1,3 +1,3 @@
 # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @opensearch-project/opensearch-core
+*   @opensearch-project/opensearch-core @reta
 

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,2 @@
+# disable blank issue creation
+blank_issues_enabled: false

.github/dependabot.yml

@@ -529,7 +529,7 @@ updates:
     package-ecosystem: gradle
     schedule:
       interval: weekly
-  - directory: /plugins/examples/painless-whitelist/
+  - directory: /plugins/examples/painless-allowlist/
     open-pull-requests-limit: 1
     package-ecosystem: gradle
     schedule:

.github/workflows/code-hygiene.yml

@@ -0,0 +1,14 @@
+name: Code Hygiene
+
+on: [pull_request]
+
+jobs:
+  linelint:
+    runs-on: ubuntu-latest
+    name: Check if all files end in newline
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Linelint
+        uses: fernandrone/[email protected]

.github/workflows/create-documentation-issue.yml

@@ -0,0 +1,41 @@
+name: Create Documentation Issue
+on:
+  pull_request:
+    types:
+      - labeled
+env:
+  PR_NUMBER: ${{ github.event.number }}
+
+jobs:
+  create-issue:
+    if: ${{ github.event.label.name == 'needs-documentation' }}
+    runs-on: ubuntu-latest
+    name: Create Documentation Issue
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Edit the issue template
+        run: |
+          echo "https://github.com/opensearch-project/OpenSearch/pull/${{ env.PR_NUMBER }}." >> ./ci/documentation/issue.md
+          
+      - name: Create Issue From File
+        id: create-issue
+        uses: peter-evans/create-issue-from-file@v4
+        with:
+          title: Add documentation related to new feature
+          content-filepath: ./ci/documentation/issue.md
+          labels: documentation
+          repository: opensearch-project/documentation-website
+          token: ${{ steps.github_app_token.outputs.token }}
+
+      - name: Print Issue
+        run: echo Created related documentation issue ${{ steps.create-issue.outputs.issue-number }}

.github/workflows/delete_backport_branch.yml

@@ -12,4 +12,4 @@ jobs:
       - name: Delete merged branch
         uses: SvanBoxel/delete-merged-branch@main
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/gradle-check.yml

@@ -0,0 +1,65 @@
+name: Gradle Check (Jenkins)
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  gradle-check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 130
+    steps:
+      - name: Setup environment variables
+        run: |
+            echo "pr_from_sha=$(jq --raw-output .pull_request.head.sha $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+            echo "pr_from_clone_url=$(jq --raw-output .pull_request.head.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+            echo "pr_to_clone_url=$(jq --raw-output .pull_request.base.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+            echo "pr_title=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+            echo "pr_number=$(jq --raw-output .pull_request.number $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+
+      - name: Checkout opensearch-build repo
+        uses: actions/checkout@v2
+        with:
+          repository: opensearch-project/opensearch-build
+          ref: main
+
+      - name: Trigger jenkins workflow to run gradle check
+        run: |
+            set -e
+            set -o pipefail
+            bash scripts/gradle/gradle-check.sh ${{ secrets.JENKINS_GRADLE_CHECK_GENERIC_WEBHOOK_TOKEN }} | tee -a gradle-check.log
+
+      - name: Setup Result Status
+        if: always()
+        run: |
+            WORKFLOW_URL=`cat gradle-check.log | grep 'WORKFLOW_URL' | awk '{print $2}'`
+            RESULT=`cat gradle-check.log | grep 'Result:' | awk '{print $2}'`
+            echo "workflow_url=$WORKFLOW_URL" >> $GITHUB_ENV
+            echo "result=$RESULT" >> $GITHUB_ENV
+
+      - name: Upload Coverage Report
+        if: success()
+        uses: codecov/codecov-action@v2
+        with:
+          files: ./codeCoverage.xml
+
+      - name: Create Comment Success
+        if: success()
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ env.pr_number }}
+          body: |
+              ### Gradle Check (Jenkins) Run Completed with:
+              * **RESULT:** ${{ env.result }} :white_check_mark:
+              * **URL:** ${{ env.workflow_url }}
+              * **CommitID:** ${{ env.pr_from_sha }}
+
+      - name: Create Comment Failure
+        if: failure()
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ env.pr_number }}
+          body: |
+              ### Gradle Check (Jenkins) Run Completed with:
+              * **RESULT:** ${{ env.result }} :x:
+              * **URL:** ${{ env.workflow_url }}
+              * **CommitID:** ${{ env.pr_from_sha }}

.github/workflows/links.yml

@@ -16,4 +16,4 @@ jobs:
           args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes
           fail: true
         env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/lucene-snapshots.yml

@@ -0,0 +1,59 @@
+# This workflow will check out, build, and publish snapshots of lucene.
+
+name: OpenSearch Lucene snapshots
+
+on:
+  workflow_dispatch:
+    # Inputs the workflow accepts.
+    inputs:
+      ref:
+        description:
+        required: false
+        default: 'main'
+
+jobs:
+  publish-snapshots:
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 17
+        uses: actions/setup-java@v2
+        with:
+          java-version: '17'
+          distribution: 'adopt'
+
+      - name: Checkout Lucene
+        uses: actions/checkout@v2
+        with:
+          repository: 'apache/lucene'
+          path: lucene
+          ref: ${{ github.event.inputs.ref }}
+
+      - name: Set hash
+        working-directory: ./lucene
+        run: |
+          echo "::set-output name=REVISION::$(git rev-parse --short HEAD)"
+        id: version
+
+      - name: Initialize gradle settings
+        working-directory: ./lucene
+        run: ./gradlew localSettings
+
+      - name: Publish Lucene to local maven repo.
+        working-directory: ./lucene
+        run: ./gradlew publishJarsPublicationToMavenLocal -Pversion.suffix=snapshot-${{ steps.version.outputs.REVISION }}
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_ROLE }}
+          aws-region: us-west-2
+
+      - name: Copy files to S3 with the aws CLI.
+        run: |
+          aws s3 cp ~/.m2/repository/org/apache/lucene/ s3://${{ secrets.LUCENE_SNAPSHOTS_BUCKET }}/snapshots/lucene/org/apache/lucene/  --recursive --no-progress

.github/workflows/version.yml

@@ -8,9 +8,15 @@ on:
 jobs:  
   build:
     runs-on: ubuntu-latest
-    env:
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+
       - uses: actions/checkout@v2
       - name: Fetch Tag and Version Information
         run: |
@@ -35,6 +41,8 @@ jobs:
       - uses: actions/checkout@v2
         with:
           ref: ${{ env.BASE }}
+          token: ${{ steps.github_app_token.outputs.token }}
+
       - name: Increment Patch Version
         run: |
           echo Incrementing $CURRENT_VERSION to $NEXT_VERSION
@@ -43,12 +51,15 @@ jobs:
           echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE
           sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java
           sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" server/src/main/java/org/opensearch/Version.java
+
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3
         with:
+          token: ${{ steps.github_app_token.outputs.token }}
           base: ${{ env.BASE }}
           branch: 'create-pull-request/patch-${{ env.BASE }}'
           commit-message: Incremented version to ${{ env.NEXT_VERSION }}
+          signoff: true
           delete-branch: true
           title: '[AUTO] Incremented version to ${{ env.NEXT_VERSION }}.'
           body: |
@@ -57,18 +68,23 @@ jobs:
       - uses: actions/checkout@v2
         with:
           ref: ${{ env.BASE_X }}
+          token: ${{ steps.github_app_token.outputs.token }}
+
       - name: Add bwc version to .X branch
         run: |
           echo Adding bwc version $NEXT_VERSION after $CURRENT_VERSION
           sed -i "s/- \"$CURRENT_VERSION\"/\0\n  - \"$NEXT_VERSION\"/g" .ci/bwcVersions
           echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE
           sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java
+
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3
         with:
+          token: ${{ steps.github_app_token.outputs.token }}
           base: ${{ env.BASE_X }}
           branch: 'create-pull-request/patch-${{ env.BASE_X }}'
           commit-message: Added bwc version ${{ env.NEXT_VERSION }}
+          signoff: true
           delete-branch: true
           title: '[AUTO] [${{ env.BASE_X }}] Added bwc version ${{ env.NEXT_VERSION }}.'
           body: |
@@ -77,18 +93,23 @@ jobs:
       - uses: actions/checkout@v2
         with:
           ref: main
+          token: ${{ steps.github_app_token.outputs.token }}
+
       - name: Add bwc version to main branch
         run: |
           echo Adding bwc version $NEXT_VERSION after $CURRENT_VERSION
           sed -i "s/- \"$CURRENT_VERSION\"/\0\n  - \"$NEXT_VERSION\"/g" .ci/bwcVersions
           echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE
           sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java
+
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3
         with:
+          token: ${{ steps.github_app_token.outputs.token }}
           base: main
           branch: 'create-pull-request/patch-main'
           commit-message: Added bwc version ${{ env.NEXT_VERSION }}
+          signoff: true
           delete-branch: true
           title: '[AUTO] [main] Added bwc version ${{ env.NEXT_VERSION }}.'
           body: |

.github/workflows/wrapper.yml

@@ -1,10 +1,10 @@
 name: Validate Gradle Wrapper
-on: [push, pull_request]
+on: [pull_request]
 
 jobs:
   validate:
     name: Validate
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: gradle/wrapper-validation-action@v1
+      - uses: gradle/wrapper-validation-action@v1

.gitignore

@@ -56,3 +56,6 @@ testfixtures_shared/
 
 # These are generated from .ci/jobs.t
 .ci/jobs/
+
+# build files generated
+doc-tools/missing-doclet/bin/