Merge remote-tracking branch 'es/main' into tsdb_synthesize_id

.buildkite/hooks/pre-command

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# On some distros, this directory ends up not readable by the `elasticsearch` user that gets created during tests
+# This fixes that
+chmod 755 ~
+
+WORKSPACE="$(pwd)"
+export WORKSPACE
+
+BUILD_NUMBER="$BUILDKITE_BUILD_NUMBER"
+export BUILD_NUMBER
+
+COMPOSE_HTTP_TIMEOUT="120"
+export COMPOSE_HTTP_TIMEOUT
+
+JOB_BRANCH="$BUILDKITE_BRANCH"
+export JOB_BRANCH
+
+GRADLEW="./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
+export GRADLEW
+
+GRADLEW_BAT="./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
+export GRADLEW_BAT
+
+export $(cat .ci/java-versions.properties | grep '=' | xargs)
+
+JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
+export JAVA_HOME
+
+JAVA11_HOME="$HOME/.java/java11"
+export JAVA11_HOME
+
+JAVA16_HOME="$HOME/.java/openjdk16"
+export JAVA16_HOME
+
+if [[ "${ES_RUNTIME_JAVA:-}" ]]; then
+  RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+  export RUNTIME_JAVA_HOME
+fi
+
+GRADLE_BUILD_CACHE_USERNAME=$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
+export GRADLE_BUILD_CACHE_USERNAME
+
+GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
+export GRADLE_BUILD_CACHE_PASSWORD
+
+BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
+export BUILDKITE_API_TOKEN
+
+if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
+  data=$(.buildkite/scripts/lucene-snapshot/get-credentials.sh)
+
+  AWS_ACCESS_KEY_ID=$(echo "$data" | jq -r .data.access_key)
+  export AWS_ACCESS_KEY_ID
+
+  AWS_SECRET_ACCESS_KEY=$(echo "$data" | jq -r .data.secret_key)
+  export AWS_SECRET_ACCESS_KEY
+
+  unset data
+fi

.buildkite/hooks/pre-command.bat

@@ -0,0 +1,21 @@
+@ECHO OFF
+
+FOR /F "tokens=* eol=#" %%i in ('type .ci\java-versions.properties') do set %%i
+
+SET JAVA_HOME=%USERPROFILE%\.java\%ES_BUILD_JAVA%
+SET JAVA16_HOME=%USERPROFILE%\.java\openjdk16
+
+SET GRADLEW=./gradlew --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+SET GRADLEW_BAT=./gradlew.bat --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+
+(if not exist "%USERPROFILE%/.gradle" mkdir "%USERPROFILE%/.gradle") && (echo. >> "%USERPROFILE%/.gradle/gradle.properties" && echo org.gradle.daemon=false >> "%USERPROFILE%/.gradle/gradle.properties")
+
+set WORKSPACE=%cd%
+set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER%
+set COMPOSE_HTTP_TIMEOUT=120
+set JOB_BRANCH=%BUILDKITE_BRANCH%
+
+set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
+set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
+
+exit /b 0

.buildkite/pipelines/lucene-snapshot/build-snapshot.yml

@@ -0,0 +1,22 @@
+steps:
+  - trigger: apache-lucene-build-snapshot
+    label: Trigger pipeline to build lucene snapshot
+    key: lucene-build
+    if: build.env("LUCENE_BUILD_ID") == null || build.env("LUCENE_BUILD_ID") == ""
+  - wait
+  - label: Upload and update lucene snapshot
+    command: .buildkite/scripts/lucene-snapshot/upload-snapshot.sh
+    timeout_in_minutes: 15
+    env:
+      USE_LUCENE_SNAPSHOT_CREDS: "true"
+      UPDATE_ES_LUCENE_SNAPSHOT: "true"
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
+  - wait
+  - trigger: "elasticsearch-lucene-snapshot-tests"
+    build:
+      branch: "${BUILDKITE_BRANCH}"
+    async: true

.buildkite/pipelines/lucene-snapshot/run-tests.yml

@@ -0,0 +1,60 @@
+steps:
+  - label: sanity-check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
+  - wait: null
+  - label: part1
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
+  - label: part2
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
+  - label: part3
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
+  - group: bwc-snapshots
+    steps:
+      - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            BWC_VERSION:
+              - 7.17.13
+              - 8.9.1
+              - 8.10.0
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: "{{matrix.BWC_VERSION}}"
+  - label: rest-compat
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/lucene-snapshot/update-branch.yml

@@ -0,0 +1,9 @@
+steps:
+  - label: Update lucene-snapshot branch
+    command: .buildkite/scripts/lucene-snapshot/update-branch.sh
+    timeout_in_minutes: 15
+  - wait
+  - trigger: "elasticsearch-lucene-snapshot-tests"
+    build:
+      branch: "${BUILDKITE_BRANCH}"
+    async: true

.buildkite/pipelines/periodic.trigger.yml

@@ -0,0 +1,16 @@
+steps:
+  - trigger: elasticsearch-periodic
+    label: Trigger periodic pipeline for main
+    async: true
+    build:
+      branch: main
+  - trigger: elasticsearch-periodic
+    label: Trigger periodic pipeline for 8.9
+    async: true
+    build:
+      branch: "8.9"
+  - trigger: elasticsearch-periodic
+    label: Trigger periodic pipeline for 7.17
+    async: true
+    build:
+      branch: "7.17"

.buildkite/pipelines/periodic.yml

@@ -0,0 +1,99 @@
+steps:
+  - group: java-fips-matrix
+    steps:
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.fips.enabled=true $$GRADLE_TASK
+        timeout_in_minutes: 180
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - openjdk17
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - bwcTestSnapshots
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+  - group: java-matrix
+    steps:
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $$GRADLE_TASK
+        timeout_in_minutes: 180
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - graalvm-ce17
+              - openjdk17
+              - openjdk18
+              - openjdk19
+              - openjdk20
+              - openjdk21
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - bwcTestSnapshots
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+  - group: packaging-tests-windows
+    steps:
+      - label: "{{matrix.image}} / packaging-tests-windows"
+        command: |
+          .\.buildkite\scripts\run-script.ps1 .\.ci\scripts\packaging-test.ps1
+        timeout_in_minutes: 180
+        matrix:
+          setup:
+            image:
+              - windows-2016
+              - windows-2019
+              - windows-2022
+        agents:
+          provider: gcp
+          image: family/brian-elasticsearch-{{matrix.image}}
+          imageProject: elastic-images-qa
+          machineType: custom-32-98304
+          diskType: pd-ssd
+          diskSizeGb: 350
+        env: {}
+  - group: platform-support-windows
+    steps:
+      - label: "{{matrix.image}} / {{matrix.GRADLE_TASK}} / platform-support-windows"
+        command: |
+          .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
+        timeout_in_minutes: 360
+        matrix:
+          setup:
+            image:
+              - windows-2016
+              - windows-2019
+              - windows-2022
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - bwcTestSnapshots
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/brian-elasticsearch-{{matrix.image}}
+          imageProject: elastic-images-qa
+          machineType: custom-32-98304
+          diskType: pd-ssd
+          diskSizeGb: 350
+        env:
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"

.buildkite/scripts/lucene-snapshot/get-credentials.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# WARNING: this script will echo the credentials to the console. It is meant to be called from another script and captured in a variable.
+# It should really only be used inside .buildkite/hooks/pre-command
+
+VAULT_ROLE_ID=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
+VAULT_SECRET_ID=$(vault read -field=secret-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
+VAULT_ADDR=https://secrets.elastic.co:8200
+
+unset VAULT_TOKEN
+VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
+export VAULT_TOKEN
+
+vault read -format=json aws-elastic/creds/lucene-snapshots

.buildkite/scripts/lucene-snapshot/remove-verification-metadata.py

@@ -0,0 +1,13 @@
+import re
+
+regex = re.compile(
+    r'<component group="org.apache.lucene" name="lucene-.*?</component>\s*',
+    re.MULTILINE | re.DOTALL,
+)
+
+with open("gradle/verification-metadata.xml", "r+") as f:
+    text = f.read()
+    text = regex.sub("", text)
+    f.seek(0)
+    f.truncate()
+    f.write(text)

.buildkite/scripts/lucene-snapshot/update-branch.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then
+  echo "Error: This script should only be run on the lucene_snapshot branch"
+  exit 1
+fi
+
+echo --- Updating lucene_snapshot branch with main
+
+git config --global user.name elasticsearchmachine
+git config --global user.email '[email protected]'
+
+git checkout lucene_snapshot
+git fetch origin main
+git merge --no-edit origin/main
+git push origin lucene_snapshot

.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then
+  echo "Error: This script should only be run on the lucene_snapshot branch"
+  exit 1
+fi
+
+echo --- Update Lucene snapshot in Elasticsearch
+
+LUCENE_SNAPSHOT_VERSION=${LUCENE_SNAPSHOT_VERSION:-}
+
+if [[ -z "$LUCENE_SNAPSHOT_VERSION" ]]; then
+  LUCENE_SNAPSHOT_VERSION=$(buildkite-agent meta-data get lucene-snapshot-version)
+fi
+
+echo "Lucene Snapshot Version: $LUCENE_SNAPSHOT_VERSION"
+
+git checkout "$BUILDKITE_BRANCH"
+git pull --ff-only origin "$BUILDKITE_BRANCH"
+
+# Replace `lucene = <version>` string in version.properties and maintain the same indentation
+sed -E "s/^(lucene *= *[^ ]*  *).*\$/\1$LUCENE_SNAPSHOT_VERSION/" build-tools-internal/version.properties > new-version.properties
+mv new-version.properties build-tools-internal/version.properties
+
+# Remove stale verification metadata, because generating them just appends new ones
+python3 .buildkite/scripts/lucene-snapshot/remove-verification-metadata.py
+./gradlew --write-verification-metadata sha256
+
+# 9.8.0-snapshot-1f25c68 -> 9.8.0
+VERSION=$(echo "$LUCENE_SNAPSHOT_VERSION" | cut -f 1 -d '-')
+VERSION_SNAKE=$(echo "$VERSION" | sed -E 's/\./_/g')
+
+sed -E "s/^(:lucene_version:  *).*\$/\1$VERSION/" docs/Versions.asciidoc > docs/Versions.asciidoc.new
+sed -E "s/^(:lucene_version_path:  *).*\$/\1$VERSION_SNAKE/" docs/Versions.asciidoc.new > docs/Versions.asciidoc
+rm -f docs/Versions.asciidoc.new
+
+if git diff-index --quiet HEAD --; then
+  echo 'No changes to commit.'
+else
+  git config --global user.name elasticsearchmachine
+  git config --global user.email '[email protected]'
+
+  git add build-tools-internal/version.properties
+  git add gradle/verification-metadata.xml
+  git add docs/Versions.asciidoc
+
+  git commit -m "[Automated] Update Lucene snapshot to $LUCENE_SNAPSHOT_VERSION"
+  git push origin "$BUILDKITE_BRANCH"
+fi