Merge branch 'master' into custom

* master: (1643 commits)
  Make DataStreamsSnapshotsIT resilient to failures because of local time. (elastic#73516)
  Upgrade netty to 4.1.63 (elastic#73011)
  [DOCS] Create a new page for dissect content in scripting docs (elastic#73437)
  Deprecate freeze index API (elastic#72618)
  [DOCS] Remove 'closed data stream' reference
  [DOCS] Update alias references (elastic#73427)
  [DOCS]  Create a new page for grok content in scripting docs (elastic#73118)
  Remove dependency on azure shadowjar since it's no longer required
  [DOCS] Update backport policy for known issues (elastic#73489)
  Shadowed dependencies should be hidden from pom dependencies (elastic#73467)
  Disable transitive dependencies when resolving bwc JDBC driver artifact (elastic#73448)
  Print full JVM implementation version at start of build (elastic#73439)
  [DOCS] Update snapshot/restore for data stream aliases (elastic#73438)
  Upgrade Azure SDK and Jackson (elastic#72833) (elastic#72995)
  [DOCS] Fix typo (elastic#73337) (elastic#73474)
  [DOCS] Fix typo (elastic#73444) (elastic#73472)
  [DOCS] Update alias security for data stream aliases (elastic#73436)
  Fix Bug with Concurrent Snapshot and Index Delete (elastic#73456)
  [DOCS] Move common scripting use cases up a level (elastic#73445)
  Add more validation for data stream aliases. (elastic#73416)
  ...

.ci/bwcVersions

@@ -32,5 +32,8 @@ BWC_VERSION:
   - "7.11.1"
   - "7.11.2"
   - "7.12.0"
+  - "7.12.1"
   - "7.13.0"
+  - "7.13.1"
+  - "7.14.0"
   - "8.0.0"

.ci/java-versions-aarch64.properties

@@ -4,6 +4,5 @@
 # build and test Elasticsearch for this branch. Valid Java versions
 # are 'java' or 'openjdk' followed by the major release number.
 
-ES_BUILD_JAVA=adoptopenjdk15
-ES_RUNTIME_JAVA=adoptopenjdk11
-GRADLE_TASK=build
+ES_BUILD_JAVA=jdk15
+ES_RUNTIME_JAVA=jdk15

.ci/java-versions.properties

@@ -6,4 +6,3 @@
 
 ES_BUILD_JAVA=openjdk15
 ES_RUNTIME_JAVA=openjdk11
-GRADLE_TASK=build

.ci/jobs.t/defaults.yml

@@ -0,0 +1,89 @@
+---
+
+##### GLOBAL METADATA
+
+- meta:
+    cluster: elasticsearch-ci
+
+##### JOB DEFAULTS
+
+- job:
+    vault:
+      url: https://secrets.elastic.co:8200
+      role_id: 1ba1ac3e-aee4-d040-d9a3-6ae23bd2b3db
+    node: "general-purpose"
+    concurrent: true
+    logrotate:
+      daysToKeep: 30
+      numToKeep: 90
+      artifactDaysToKeep: 7
+    parameters:
+      - string:
+          name: branch_specifier
+          default: "refs/heads/%BRANCH%"
+          description: "the Git branch specifier to build (<branchName>, <tagName>, <commitId>, etc.)\n"
+    scm:
+      - git:
+          name: origin
+          # master node jenkins user ~/.ssh
+          credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+          reference-repo: "/var/lib/jenkins/.git-references/elasticsearch.git"
+          branches:
+            - "${branch_specifier}"
+          url: "https://github.com/elastic/elasticsearch.git"
+          basedir: ""
+          wipe-workspace: true
+    triggers: []
+    wrappers:
+      - timeout:
+          type: absolute
+          timeout: 480
+          fail: true
+      - ansicolor
+      - timestamps
+      - gradle-build-scan
+      - inject-passwords:
+          global: false
+          job-passwords:
+            - name: VAULT_ADDR
+              password: https://secrets.elastic.co:8200
+          mask-password-params: true
+    properties:
+      - github:
+          url: https://github.com/elastic/elasticsearch/
+      - inject:
+          properties-content: |
+            JOB_BRANCH=%BRANCH%
+            HOME=$JENKINS_HOME
+            GRADLEW=./gradlew --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+            GRADLEW_BAT=./gradlew.bat --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+    publishers:
+      - junit:
+          results: "**/*Junit/*.xml, **/test-results/*/*.xml"
+          keep-long-stdio: true
+          allow-empty-results: true
+      # Upload additional logs
+      - google-cloud-storage:
+          credentials-id: 'elasticsearch-ci-gcs-plugin'
+          uploads:
+            - classic:
+                file-pattern: 'build/*.tar.bz2'
+                storage-location: 'gs://elasticsearch-ci-artifacts/jobs/$JOB_NAME'
+                share-publicly: false
+                upload-for-failed-jobs: true
+                show-inline: true
+      # Notify homer
+      - postbuildscript:
+          builders:
+            - role: SLAVE
+              build-on:
+                - SUCCESS
+                - FAILURE
+                - UNSTABLE
+              build-steps:
+                - http-request:
+                    url: https://homer.app.elstc.co/webhook/jenkins/build-finished
+                    mode: GET
+                    custom-headers:
+                      - name: X-Jenkins-Build
+                        value: ${BUILD_URL}

.ci/jobs.t/elastic+elasticsearch+branch-consistency.yml

@@ -0,0 +1,15 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+branch-consistency
+    display-name: "elastic / elasticsearch # %BRANCH% - branch consistency"
+    description: Testing of the Elasticsearch master branch version consistency.
+    triggers:
+      - timed: "H 7 * * *"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh branchConsistency

.ci/jobs.t/elastic+elasticsearch+branch-protection.yml

@@ -0,0 +1,20 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+branch-protection
+    display-name: "elastic / elasticsearch # %BRANCH% - branch protection"
+    description: Elasticsearch %BRANCH% branch protection.
+    node: master
+    triggers:
+      - timed: "H 7 * * *"
+    scm: []
+    parameters: []
+    builders:
+      - shell: |
+          #!/bin/bash
+          set +x
+          STATUS=$(curl -s https://api.github.com/repos/elastic/elasticsearch/branches/%BRANCH% | jq '.protected')
+          echo "Branch %BRANCH% protection status is: $STATUS"
+          if [[ "$STATUS" == "false" ]]; then
+            echo "Development branch %BRANCH% is not set as protected in GitHub but should be."
+            exit 1
+          fi

.ci/jobs.t/elastic+elasticsearch+folder+pull-request.yml

@@ -0,0 +1,4 @@
+- job:
+    name: elastic+elasticsearch+%BRANCH%+pull-request
+    display-name: Pull Requests
+    project-type: folder

.ci/jobs.t/elastic+elasticsearch+folder+triggers.yml

@@ -0,0 +1,4 @@
+- job:
+    name: elastic+elasticsearch+%BRANCH%+triggers
+    display-name: Periodic Triggers
+    project-type: folder

.ci/jobs.t/elastic+elasticsearch+intake+multijob+bwc.yml

@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake bwc"
+  - job-description: Elasticsearch %BRANCH% branch intake backwards compatibility checks.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files bwcTestSnapshots"

.ci/jobs.t/elastic+elasticsearch+intake+multijob+part1.yml

@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+part1
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake part 1"
+  - job-description: Elasticsearch %BRANCH% branch intake check part 1.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1"

.ci/jobs.t/elastic+elasticsearch+intake+multijob+part2.yml

@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+part2
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake part 2"
+  - job-description: Elasticsearch %BRANCH% branch intake check part 2.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2"

.ci/jobs.t/elastic+elasticsearch+intake+multijob+rest-compat.yml

@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+rest-compat
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake rest compatibility"
+  - job-description: Elasticsearch %BRANCH% branch intake REST compatibility checks.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat"

.ci/jobs.t/elastic+elasticsearch+intake+multijob+sanity-check.yml

@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+sanity-check
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake sanity check"
+  - job-description: Elasticsearch %BRANCH% branch intake sanity check.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit"

.ci/jobs.t/elastic+elasticsearch+intake+multijob+update-last-good-commit.yml

@@ -0,0 +1,11 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+intake+multijob+update-last-good-commit
+    display-name: "elastic / elasticsearch # %BRANCH% - update last good commit"
+    description: Elasticsearch %BRANCH% branch update last good commit in build-stats.
+    node: light
+    properties: []
+    builders:
+      - shell: |
+          #!/usr/local/bin/runbld --job-name elastic+elasticsearch+%BRANCH%+git+push
+          /usr/bin/true

.ci/jobs.t/elastic+elasticsearch+intake.yml

@@ -0,0 +1,59 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+intake
+    display-name: "elastic / elasticsearch # %BRANCH% - intake"
+    description: "Testing of the Elasticsearch %BRANCH% branch on every push.\n"
+    project-type: multijob
+    node: master
+    vault: []
+    triggers:
+      # We use this trigger instead of the provided "github" webhook trigger because it's more robust.
+      # Here we only trigger builds for pushes to the corresponding branch, rather than a push to any branch of the
+      # configured git repository. This avoids duplicate builds being triggered when pushes to multiple branches are
+      # done in quick succession.
+      - generic-webhook-trigger:
+          post-content-params:
+            - type: JSONPath
+              key: ref
+              value: '$.ref'
+          regex-filter-text: '$ref'
+          regex-filter-expression: "^refs/heads/%BRANCH%$"
+          cause: Push to GitHub (refs/heads/%BRANCH%)
+          silent-response: true
+    scm:
+      - git:
+          wipe-workspace: false
+    builders:
+      - multijob:
+          name: Sanity Check
+          projects:
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+sanity-check
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+      - multijob:
+          name: Verification
+          projects:
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+part1
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+part2
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+rest-compat
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+      - multijob:
+          name: Update last good commit
+          projects:
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+update-last-good-commit
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true

.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-unix.yml

@@ -0,0 +1,36 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+multijob+packaging-tests-unix
+    display-name: "elastic / elasticsearch # %BRANCH% - unix packaging tests"
+    description: "Testing of the Elasticsearch %BRANCH% branch unix packaging test support matrix.\n"
+    project-type: matrix
+    node: master
+    scm:
+      - git:
+          wipe-workspace: false
+    axes:
+      - axis:
+          type: label-expression
+          name: os
+          values:
+            - centos-7-packaging
+            - centos-8-packaging
+            - debian-9-packaging
+            - debian-10-packaging
+            - fedora-32-packaging
+            - opensuse-15-1-packaging
+            - oraclelinux-7-packaging
+            - oraclelinux-8-packaging
+            - sles-12-packaging
+            - sles-15-packaging
+            - ubuntu-18.04-packaging
+            - ubuntu-20.04-packaging
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          ./.ci/os.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ destructivePackagingTest

.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-upgrade.yml

@@ -0,0 +1,31 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+multijob+packaging-tests-upgrade
+    display-name: "elastic / elasticsearch # %BRANCH% - packaging upgrade tests"
+    description: "Testing of the Elasticsearch %BRANCH% branch packaging test upgrade support matrix.\n"
+    project-type: matrix
+    node: master
+    scm:
+      - git:
+          wipe-workspace: false
+    axes:
+      - axis:
+          type: label-expression
+          name: os
+          values:
+            - centos-8-packaging
+            - ubuntu-20.04-packaging
+      - axis:
+          type: yaml
+          filename: ".ci/bwcVersions"
+          name: BWC_VERSION
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA15_HOME=$HOME/.java/openjdk15
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          ./.ci/os.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ destructiveDistroUpgradeTest.v$BWC_VERSION